38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
274 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
286 if (!Subtarget.hasStdExtZbb())
290 if (Subtarget.hasStdExtZbb()) {
298 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
303 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
307 }
else if (Subtarget.
is64Bit()) {
317 if (!Subtarget.hasStdExtM()) {
323 }
else if (Subtarget.
is64Bit()) {
326 {MVT::i8, MVT::i16, MVT::i32},
Custom);
343 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
346 }
else if (Subtarget.hasVendorXTHeadBb()) {
350 }
else if (Subtarget.hasVendorXCVbitmanip()) {
361 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
362 Subtarget.hasVendorXTHeadBb())
367 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
368 Subtarget.hasVendorXTHeadBb())
373 if (Subtarget.hasVendorXCVbitmanip()) {
381 if (Subtarget.hasStdExtZbb()) {
394 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
400 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
401 Subtarget.hasVendorXCVbitmanip()) {
408 if (!Subtarget.hasStdExtZbb())
420 !Subtarget.hasShortForwardBranchOpt())
424 if (Subtarget.hasShortForwardBranchOpt())
427 if (!Subtarget.hasVendorXTHeadCondMov()) {
433 static const unsigned FPLegalNodeTypes[] = {
446 static const unsigned FPOpToExpand[] = {
450 static const unsigned FPRndMode[] = {
457 static const unsigned ZfhminZfbfminPromoteOps[] = {
468 if (Subtarget.hasStdExtZfbfmin()) {
551 if (Subtarget.hasStdExtZfa()) {
568 if (Subtarget.hasStdExtZfa()) {
645 if (Subtarget.hasStdExtZicbop()) {
649 if (Subtarget.hasStdExtA()) {
651 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
655 }
else if (Subtarget.hasForcedAtomics()) {
676 {MVT::i8, MVT::i16},
Custom);
687 static const unsigned IntegerVPOps[] = {
688 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
689 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
690 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
691 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
692 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
693 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
694 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
695 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
696 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
697 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
698 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
699 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
700 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
703 static const unsigned FloatingPointVPOps[] = {
704 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
705 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
706 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
707 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
708 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
709 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
710 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
711 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
712 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
713 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
714 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
715 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
716 ISD::EXPERIMENTAL_VP_SPLICE};
718 static const unsigned IntegerVecReduceOps[] = {
723 static const unsigned FloatingPointVecReduceOps[] = {
736 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
737 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
738 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
742 for (
MVT VT : BoolVecVTs) {
769 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
793 ISD::VP_TRUNCATE, ISD::VP_SETCC},
809 for (
MVT VT : IntVecVTs) {
820 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
867 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
868 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
892 if (Subtarget.hasStdExtZvkb()) {
900 if (Subtarget.hasStdExtZvbb()) {
904 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
910 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
919 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
939 static const unsigned ZvfhminPromoteOps[] = {
949 static const unsigned ZvfhminPromoteVPOps[] = {
950 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
951 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
952 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
953 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
954 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
955 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
956 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
957 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
961 const auto SetCommonVFPActions = [&](
MVT VT) {
1006 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1007 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1038 const auto SetCommonVFPExtLoadTruncStoreActions =
1040 for (
auto SmallVT : SmallerVTs) {
1047 for (
MVT VT : F16VecVTs) {
1050 SetCommonVFPActions(VT);
1053 for (
MVT VT : F16VecVTs) {
1064 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1075 if (VT == MVT::nxv32f16) {
1088 for (
MVT VT : F32VecVTs) {
1091 SetCommonVFPActions(VT);
1092 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1097 for (
MVT VT : F64VecVTs) {
1100 SetCommonVFPActions(VT);
1101 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1102 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1108 if (!useRVVForFixedLengthVectorVT(VT))
1151 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1178 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1200 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1201 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1238 if (Subtarget.hasStdExtZvkb())
1241 if (Subtarget.hasStdExtZvbb()) {
1263 if (!useRVVForFixedLengthVectorVT(VT))
1284 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1316 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1317 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1368 if (Subtarget.hasStdExtA()) {
1374 if (Subtarget.hasForcedAtomics()) {
1384 if (Subtarget.hasVendorXTHeadMemIdx()) {
1416 if (Subtarget.hasStdExtZbb())
1419 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1422 if (Subtarget.hasStdExtZbkb())
1434 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1437 if (Subtarget.hasVendorXTHeadMemPair())
1460MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1465bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1467 bool IsScalable)
const {
1474 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1491bool RISCVTargetLowering::shouldExpandCttzElements(
EVT VT)
const {
1499 unsigned Intrinsic)
const {
1500 auto &
DL =
I.getModule()->getDataLayout();
1502 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1503 bool IsUnitStrided,
bool UsePtrVal =
false) {
1508 Info.ptrVal =
I.getArgOperand(PtrOp);
1510 Info.fallbackAddressSpace =
1511 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1515 MemTy =
I.getArgOperand(0)->getType();
1518 MemTy =
I.getType();
1533 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1537 switch (Intrinsic) {
1540 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1541 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1542 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1543 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1544 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1545 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1546 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1547 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1548 case Intrinsic::riscv_masked_cmpxchg_i32:
1550 Info.memVT = MVT::i32;
1551 Info.ptrVal =
I.getArgOperand(0);
1557 case Intrinsic::riscv_masked_strided_load:
1558 return SetRVVLoadStoreInfo( 1,
false,
1560 case Intrinsic::riscv_masked_strided_store:
1561 return SetRVVLoadStoreInfo( 1,
true,
1563 case Intrinsic::riscv_seg2_load:
1564 case Intrinsic::riscv_seg3_load:
1565 case Intrinsic::riscv_seg4_load:
1566 case Intrinsic::riscv_seg5_load:
1567 case Intrinsic::riscv_seg6_load:
1568 case Intrinsic::riscv_seg7_load:
1569 case Intrinsic::riscv_seg8_load:
1570 return SetRVVLoadStoreInfo( 0,
false,
1572 case Intrinsic::riscv_seg2_store:
1573 case Intrinsic::riscv_seg3_store:
1574 case Intrinsic::riscv_seg4_store:
1575 case Intrinsic::riscv_seg5_store:
1576 case Intrinsic::riscv_seg6_store:
1577 case Intrinsic::riscv_seg7_store:
1578 case Intrinsic::riscv_seg8_store:
1580 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1583 case Intrinsic::riscv_vle:
1584 case Intrinsic::riscv_vle_mask:
1585 case Intrinsic::riscv_vleff:
1586 case Intrinsic::riscv_vleff_mask:
1587 return SetRVVLoadStoreInfo( 1,
1591 case Intrinsic::riscv_vse:
1592 case Intrinsic::riscv_vse_mask:
1593 return SetRVVLoadStoreInfo( 1,
1597 case Intrinsic::riscv_vlse:
1598 case Intrinsic::riscv_vlse_mask:
1599 case Intrinsic::riscv_vloxei:
1600 case Intrinsic::riscv_vloxei_mask:
1601 case Intrinsic::riscv_vluxei:
1602 case Intrinsic::riscv_vluxei_mask:
1603 return SetRVVLoadStoreInfo( 1,
1606 case Intrinsic::riscv_vsse:
1607 case Intrinsic::riscv_vsse_mask:
1608 case Intrinsic::riscv_vsoxei:
1609 case Intrinsic::riscv_vsoxei_mask:
1610 case Intrinsic::riscv_vsuxei:
1611 case Intrinsic::riscv_vsuxei_mask:
1612 return SetRVVLoadStoreInfo( 1,
1615 case Intrinsic::riscv_vlseg2:
1616 case Intrinsic::riscv_vlseg3:
1617 case Intrinsic::riscv_vlseg4:
1618 case Intrinsic::riscv_vlseg5:
1619 case Intrinsic::riscv_vlseg6:
1620 case Intrinsic::riscv_vlseg7:
1621 case Intrinsic::riscv_vlseg8:
1622 case Intrinsic::riscv_vlseg2ff:
1623 case Intrinsic::riscv_vlseg3ff:
1624 case Intrinsic::riscv_vlseg4ff:
1625 case Intrinsic::riscv_vlseg5ff:
1626 case Intrinsic::riscv_vlseg6ff:
1627 case Intrinsic::riscv_vlseg7ff:
1628 case Intrinsic::riscv_vlseg8ff:
1629 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1632 case Intrinsic::riscv_vlseg2_mask:
1633 case Intrinsic::riscv_vlseg3_mask:
1634 case Intrinsic::riscv_vlseg4_mask:
1635 case Intrinsic::riscv_vlseg5_mask:
1636 case Intrinsic::riscv_vlseg6_mask:
1637 case Intrinsic::riscv_vlseg7_mask:
1638 case Intrinsic::riscv_vlseg8_mask:
1639 case Intrinsic::riscv_vlseg2ff_mask:
1640 case Intrinsic::riscv_vlseg3ff_mask:
1641 case Intrinsic::riscv_vlseg4ff_mask:
1642 case Intrinsic::riscv_vlseg5ff_mask:
1643 case Intrinsic::riscv_vlseg6ff_mask:
1644 case Intrinsic::riscv_vlseg7ff_mask:
1645 case Intrinsic::riscv_vlseg8ff_mask:
1646 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1649 case Intrinsic::riscv_vlsseg2:
1650 case Intrinsic::riscv_vlsseg3:
1651 case Intrinsic::riscv_vlsseg4:
1652 case Intrinsic::riscv_vlsseg5:
1653 case Intrinsic::riscv_vlsseg6:
1654 case Intrinsic::riscv_vlsseg7:
1655 case Intrinsic::riscv_vlsseg8:
1656 case Intrinsic::riscv_vloxseg2:
1657 case Intrinsic::riscv_vloxseg3:
1658 case Intrinsic::riscv_vloxseg4:
1659 case Intrinsic::riscv_vloxseg5:
1660 case Intrinsic::riscv_vloxseg6:
1661 case Intrinsic::riscv_vloxseg7:
1662 case Intrinsic::riscv_vloxseg8:
1663 case Intrinsic::riscv_vluxseg2:
1664 case Intrinsic::riscv_vluxseg3:
1665 case Intrinsic::riscv_vluxseg4:
1666 case Intrinsic::riscv_vluxseg5:
1667 case Intrinsic::riscv_vluxseg6:
1668 case Intrinsic::riscv_vluxseg7:
1669 case Intrinsic::riscv_vluxseg8:
1670 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1673 case Intrinsic::riscv_vlsseg2_mask:
1674 case Intrinsic::riscv_vlsseg3_mask:
1675 case Intrinsic::riscv_vlsseg4_mask:
1676 case Intrinsic::riscv_vlsseg5_mask:
1677 case Intrinsic::riscv_vlsseg6_mask:
1678 case Intrinsic::riscv_vlsseg7_mask:
1679 case Intrinsic::riscv_vlsseg8_mask:
1680 case Intrinsic::riscv_vloxseg2_mask:
1681 case Intrinsic::riscv_vloxseg3_mask:
1682 case Intrinsic::riscv_vloxseg4_mask:
1683 case Intrinsic::riscv_vloxseg5_mask:
1684 case Intrinsic::riscv_vloxseg6_mask:
1685 case Intrinsic::riscv_vloxseg7_mask:
1686 case Intrinsic::riscv_vloxseg8_mask:
1687 case Intrinsic::riscv_vluxseg2_mask:
1688 case Intrinsic::riscv_vluxseg3_mask:
1689 case Intrinsic::riscv_vluxseg4_mask:
1690 case Intrinsic::riscv_vluxseg5_mask:
1691 case Intrinsic::riscv_vluxseg6_mask:
1692 case Intrinsic::riscv_vluxseg7_mask:
1693 case Intrinsic::riscv_vluxseg8_mask:
1694 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1697 case Intrinsic::riscv_vsseg2:
1698 case Intrinsic::riscv_vsseg3:
1699 case Intrinsic::riscv_vsseg4:
1700 case Intrinsic::riscv_vsseg5:
1701 case Intrinsic::riscv_vsseg6:
1702 case Intrinsic::riscv_vsseg7:
1703 case Intrinsic::riscv_vsseg8:
1704 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1707 case Intrinsic::riscv_vsseg2_mask:
1708 case Intrinsic::riscv_vsseg3_mask:
1709 case Intrinsic::riscv_vsseg4_mask:
1710 case Intrinsic::riscv_vsseg5_mask:
1711 case Intrinsic::riscv_vsseg6_mask:
1712 case Intrinsic::riscv_vsseg7_mask:
1713 case Intrinsic::riscv_vsseg8_mask:
1714 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1717 case Intrinsic::riscv_vssseg2:
1718 case Intrinsic::riscv_vssseg3:
1719 case Intrinsic::riscv_vssseg4:
1720 case Intrinsic::riscv_vssseg5:
1721 case Intrinsic::riscv_vssseg6:
1722 case Intrinsic::riscv_vssseg7:
1723 case Intrinsic::riscv_vssseg8:
1724 case Intrinsic::riscv_vsoxseg2:
1725 case Intrinsic::riscv_vsoxseg3:
1726 case Intrinsic::riscv_vsoxseg4:
1727 case Intrinsic::riscv_vsoxseg5:
1728 case Intrinsic::riscv_vsoxseg6:
1729 case Intrinsic::riscv_vsoxseg7:
1730 case Intrinsic::riscv_vsoxseg8:
1731 case Intrinsic::riscv_vsuxseg2:
1732 case Intrinsic::riscv_vsuxseg3:
1733 case Intrinsic::riscv_vsuxseg4:
1734 case Intrinsic::riscv_vsuxseg5:
1735 case Intrinsic::riscv_vsuxseg6:
1736 case Intrinsic::riscv_vsuxseg7:
1737 case Intrinsic::riscv_vsuxseg8:
1738 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1741 case Intrinsic::riscv_vssseg2_mask:
1742 case Intrinsic::riscv_vssseg3_mask:
1743 case Intrinsic::riscv_vssseg4_mask:
1744 case Intrinsic::riscv_vssseg5_mask:
1745 case Intrinsic::riscv_vssseg6_mask:
1746 case Intrinsic::riscv_vssseg7_mask:
1747 case Intrinsic::riscv_vssseg8_mask:
1748 case Intrinsic::riscv_vsoxseg2_mask:
1749 case Intrinsic::riscv_vsoxseg3_mask:
1750 case Intrinsic::riscv_vsoxseg4_mask:
1751 case Intrinsic::riscv_vsoxseg5_mask:
1752 case Intrinsic::riscv_vsoxseg6_mask:
1753 case Intrinsic::riscv_vsoxseg7_mask:
1754 case Intrinsic::riscv_vsoxseg8_mask:
1755 case Intrinsic::riscv_vsuxseg2_mask:
1756 case Intrinsic::riscv_vsuxseg3_mask:
1757 case Intrinsic::riscv_vsuxseg4_mask:
1758 case Intrinsic::riscv_vsuxseg5_mask:
1759 case Intrinsic::riscv_vsuxseg6_mask:
1760 case Intrinsic::riscv_vsuxseg7_mask:
1761 case Intrinsic::riscv_vsuxseg8_mask:
1762 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1799 return isInt<12>(Imm);
1803 return isInt<12>(Imm);
1816 return (SrcBits == 64 && DestBits == 32);
1827 return (SrcBits == 64 && DestBits == 32);
1834 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1835 EVT MemVT = LD->getMemoryVT();
1836 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1846 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1854 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1858 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1859 Subtarget.hasVendorXCVbitmanip();
1870 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1875 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1879 EVT VT =
Y.getValueType();
1885 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1886 !isa<ConstantSDNode>(
Y);
1891 if (Subtarget.hasStdExtZbs())
1892 return X.getValueType().isScalarInteger();
1893 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1895 if (Subtarget.hasVendorXTHeadBs())
1896 return C !=
nullptr;
1898 return C &&
C->getAPIntValue().ule(10);
1918 if (BitSize > Subtarget.
getXLen())
1922 int64_t Val = Imm.getSExtValue();
1930 if (!Subtarget.enableUnalignedScalarMem())
1946 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1953 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1957 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1969 case Instruction::Add:
1970 case Instruction::Sub:
1971 case Instruction::Mul:
1972 case Instruction::And:
1973 case Instruction::Or:
1974 case Instruction::Xor:
1975 case Instruction::FAdd:
1976 case Instruction::FSub:
1977 case Instruction::FMul:
1978 case Instruction::FDiv:
1979 case Instruction::ICmp:
1980 case Instruction::FCmp:
1982 case Instruction::Shl:
1983 case Instruction::LShr:
1984 case Instruction::AShr:
1985 case Instruction::UDiv:
1986 case Instruction::SDiv:
1987 case Instruction::URem:
1988 case Instruction::SRem:
1989 return Operand == 1;
2003 auto *II = dyn_cast<IntrinsicInst>(
I);
2007 switch (II->getIntrinsicID()) {
2008 case Intrinsic::fma:
2009 case Intrinsic::vp_fma:
2010 return Operand == 0 || Operand == 1;
2011 case Intrinsic::vp_shl:
2012 case Intrinsic::vp_lshr:
2013 case Intrinsic::vp_ashr:
2014 case Intrinsic::vp_udiv:
2015 case Intrinsic::vp_sdiv:
2016 case Intrinsic::vp_urem:
2017 case Intrinsic::vp_srem:
2018 case Intrinsic::ssub_sat:
2019 case Intrinsic::vp_ssub_sat:
2020 case Intrinsic::usub_sat:
2021 case Intrinsic::vp_usub_sat:
2022 return Operand == 1;
2024 case Intrinsic::vp_add:
2025 case Intrinsic::vp_mul:
2026 case Intrinsic::vp_and:
2027 case Intrinsic::vp_or:
2028 case Intrinsic::vp_xor:
2029 case Intrinsic::vp_fadd:
2030 case Intrinsic::vp_fmul:
2031 case Intrinsic::vp_icmp:
2032 case Intrinsic::vp_fcmp:
2033 case Intrinsic::smin:
2034 case Intrinsic::vp_smin:
2035 case Intrinsic::umin:
2036 case Intrinsic::vp_umin:
2037 case Intrinsic::smax:
2038 case Intrinsic::vp_smax:
2039 case Intrinsic::umax:
2040 case Intrinsic::vp_umax:
2041 case Intrinsic::sadd_sat:
2042 case Intrinsic::vp_sadd_sat:
2043 case Intrinsic::uadd_sat:
2044 case Intrinsic::vp_uadd_sat:
2046 case Intrinsic::vp_sub:
2047 case Intrinsic::vp_fsub:
2048 case Intrinsic::vp_fdiv:
2049 return Operand == 0 || Operand == 1;
2070 if (!Subtarget.sinkSplatOperands())
2073 for (
auto OpIdx :
enumerate(
I->operands())) {
2077 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2079 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2088 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2093 for (
Use &U :
Op->uses()) {
2145 if (!Subtarget.hasStdExtZfa())
2146 return std::make_pair(-1,
false);
2148 bool IsSupportedVT =
false;
2149 if (VT == MVT::f16) {
2150 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2151 }
else if (VT == MVT::f32) {
2152 IsSupportedVT =
true;
2153 }
else if (VT == MVT::f64) {
2154 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2155 IsSupportedVT =
true;
2159 return std::make_pair(-1,
false);
2162 if (
Index < 0 && Imm.isNegative())
2166 return std::make_pair(
Index,
false);
2170 bool ForCodeSize)
const {
2171 bool IsLegalVT =
false;
2174 else if (VT == MVT::f32)
2176 else if (VT == MVT::f64)
2178 else if (VT == MVT::bf16)
2179 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2191 return Imm.isZero();
2195 if (Imm.isNegZero())
2208 unsigned Index)
const {
2221 if (EltVT == MVT::i1)
2234 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2241 if ((ResElts * 2) != SrcElts)
2285 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2287 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2290 IntermediateVT = MVT::i64;
2293 RegisterVT = MVT::i64;
2308 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2314 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2327 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2328 int64_t
C = RHSC->getSExtValue();
2370 switch (KnownSize) {
2398 return RISCV::VRRegClassID;
2400 return RISCV::VRM2RegClassID;
2402 return RISCV::VRM4RegClassID;
2404 return RISCV::VRM8RegClassID;
2414 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2415 "Unexpected subreg numbering");
2416 return RISCV::sub_vrm1_0 +
Index;
2419 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2420 "Unexpected subreg numbering");
2421 return RISCV::sub_vrm2_0 +
Index;
2424 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2425 "Unexpected subreg numbering");
2426 return RISCV::sub_vrm4_0 +
Index;
2433 return RISCV::VRRegClassID;
2442std::pair<unsigned, unsigned>
2444 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2446 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2447 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2448 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2449 "Register classes not ordered");
2458 unsigned SubRegIdx = RISCV::NoSubRegister;
2459 for (
const unsigned RCID :
2460 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2461 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2465 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2470 return {SubRegIdx, InsertExtractIdx};
2475bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2504unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2511 "Unexpected opcode");
2513 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2515 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2518 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2588bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2589 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2598 "Expected legal fixed length vector!");
2601 unsigned MaxELen = Subtarget.
getELen();
2634 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2641 "Expected to convert into a scalable vector!");
2642 assert(V.getValueType().isFixedLengthVector() &&
2643 "Expected a fixed length vector operand!");
2653 "Expected to convert into a fixed length vector!");
2654 assert(V.getValueType().isScalableVector() &&
2655 "Expected a scalable vector operand!");
2683 const auto [MinVLMAX, MaxVLMAX] =
2685 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2691static std::pair<SDValue, SDValue>
2700static std::pair<SDValue, SDValue>
2713static std::pair<SDValue, SDValue>
2730std::pair<unsigned, unsigned>
2746 return std::make_pair(MinVLMAX, MaxVLMAX);
2758 EVT VT,
unsigned DefinedValues)
const {
2772 std::tie(LMul, Fractional) =
2775 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2777 Cost = (LMul * DLenFactor);
2822 MVT DstVT =
Op.getSimpleValueType();
2823 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2831 Src.getValueType() == MVT::bf16) {
2838 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2846 Opc,
DL, DstVT, Src,
2860 MVT SrcVT = Src.getSimpleValueType();
2866 if (SatVT != DstEltVT)
2870 if (SrcEltSize > (2 * DstEltSize))
2873 MVT DstContainerVT = DstVT;
2874 MVT SrcContainerVT = SrcVT;
2880 "Expected same element count");
2889 {Src, Src, DAG.getCondCode(ISD::SETNE),
2890 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2894 if (DstEltSize > (2 * SrcEltSize)) {
2908 Res, DAG.
getUNDEF(DstContainerVT), VL);
2920 case ISD::VP_FROUNDEVEN:
2924 case ISD::VP_FROUNDTOZERO:
2928 case ISD::VP_FFLOOR:
2936 case ISD::VP_FROUND:
2952 MVT VT =
Op.getSimpleValueType();
2959 MVT ContainerVT = VT;
2966 if (
Op->isVPOpcode()) {
2967 Mask =
Op.getOperand(1);
2971 VL =
Op.getOperand(2);
2993 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3007 switch (
Op.getOpcode()) {
3013 case ISD::VP_FFLOOR:
3016 case ISD::VP_FROUND:
3017 case ISD::VP_FROUNDEVEN:
3018 case ISD::VP_FROUNDTOZERO: {
3034 case ISD::VP_FNEARBYINT:
3047 Src, Src, Mask, VL);
3062 MVT VT =
Op.getSimpleValueType();
3066 MVT ContainerVT = VT;
3078 MVT MaskVT = Mask.getSimpleValueType();
3081 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3082 DAG.getUNDEF(MaskVT), Mask, VL});
3086 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
3087 Chain = Src.getValue(1);
3103 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3115 switch (
Op.getOpcode()) {
3126 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3132 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3136 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3145 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3146 Truncated, Mask, VL);
3152 Src, Src, Mask, VL);
3162 MVT VT =
Op.getSimpleValueType();
3190 MVT VT =
Op.getSimpleValueType();
3195 MVT ContainerVT = VT;
3217 if (
Merge.isUndef())
3229 if (
Merge.isUndef())
3238 "Unexpected vector MVT");
3266 return std::nullopt;
3281 unsigned EltSizeInBits) {
3284 return std::nullopt;
3285 bool IsInteger =
Op.getValueType().isInteger();
3287 std::optional<unsigned> SeqStepDenom;
3288 std::optional<int64_t> SeqStepNum, SeqAddend;
3289 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3290 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3295 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3297 if (Elt.isUndef()) {
3298 Elts[
Idx] = std::nullopt;
3302 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3307 return std::nullopt;
3308 Elts[
Idx] = *ExactInteger;
3321 unsigned IdxDiff =
Idx - PrevElt->second;
3322 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3330 int64_t Remainder = ValDiff % IdxDiff;
3332 if (Remainder != ValDiff) {
3335 return std::nullopt;
3341 SeqStepNum = ValDiff;
3342 else if (ValDiff != SeqStepNum)
3343 return std::nullopt;
3346 SeqStepDenom = IdxDiff;
3347 else if (IdxDiff != *SeqStepDenom)
3348 return std::nullopt;
3352 if (!PrevElt || PrevElt->first != *Elt)
3353 PrevElt = std::make_pair(*Elt,
Idx);
3357 if (!SeqStepNum || !SeqStepDenom)
3358 return std::nullopt;
3366 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3367 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3370 else if (Addend != SeqAddend)
3371 return std::nullopt;
3374 assert(SeqAddend &&
"Must have an addend if we have a step");
3376 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3397 MVT ContainerVT = VT;
3425 MVT VT =
Op.getSimpleValueType();
3437 unsigned MostCommonCount = 0;
3439 unsigned NumUndefElts =
3447 unsigned NumScalarLoads = 0;
3453 ValueCounts.
insert(std::make_pair(V, 0));
3454 unsigned &Count = ValueCounts[V];
3456 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3457 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3462 if (++Count >= MostCommonCount) {
3464 MostCommonCount = Count;
3468 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3469 unsigned NumDefElts = NumElts - NumUndefElts;
3470 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3476 ((MostCommonCount > DominantValueCountThreshold) ||
3489 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3490 LastOp != DominantValue) {
3499 Processed.insert(LastOp);
3504 const SDValue &V = OpIdx.value();
3505 if (V.isUndef() || !Processed.insert(V).second)
3507 if (ValueCounts[V] == 1) {
3516 return DAG.getConstant(V == V1, DL, XLenVT);
3532 MVT VT =
Op.getSimpleValueType();
3562 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3563 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3571 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3572 MVT IntegerViaVecVT =
3577 unsigned BitPos = 0, IntegerEltIdx = 0;
3580 for (
unsigned I = 0;
I < NumElts;) {
3582 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3583 Bits |= ((
uint64_t)BitValue << BitPos);
3589 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3590 if (NumViaIntegerBits <= 32)
3591 Bits = SignExtend64<32>(Bits);
3593 Elts[IntegerEltIdx] = Elt;
3602 if (NumElts < NumViaIntegerBits) {
3606 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3634 int64_t StepNumerator = SimpleVID->StepNumerator;
3635 unsigned StepDenominator = SimpleVID->StepDenominator;
3636 int64_t Addend = SimpleVID->Addend;
3638 assert(StepNumerator != 0 &&
"Invalid step");
3639 bool Negate =
false;
3640 int64_t SplatStepVal = StepNumerator;
3644 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3646 Negate = StepNumerator < 0;
3648 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3655 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3656 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3658 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3661 MVT VIDContainerVT =
3669 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3670 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3672 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3674 if (StepDenominator != 1) {
3679 if (Addend != 0 || Negate) {
3698 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3699 "Unexpected sequence type");
3703 unsigned ViaVecLen =
3707 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3710 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3711 const auto &SeqV = OpIdx.value();
3712 if (!SeqV.isUndef())
3714 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3719 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3720 SplatValue = SignExtend64<32>(SplatValue);
3742 const auto *BV = cast<BuildVectorSDNode>(
Op);
3745 BV->getRepeatedSequence(Sequence) &&
3746 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3747 unsigned SeqLen = Sequence.size();
3749 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3750 ViaIntVT == MVT::i64) &&
3751 "Unexpected sequence type");
3756 const unsigned RequiredVL = NumElts / SeqLen;
3757 const unsigned ViaVecLen =
3759 NumElts : RequiredVL;
3762 unsigned EltIdx = 0;
3763 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3767 for (
const auto &SeqV : Sequence) {
3768 if (!SeqV.isUndef())
3770 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3776 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3777 SplatValue = SignExtend64<32>(SplatValue);
3783 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3784 "Unexpected bitcast sequence");
3785 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3788 MVT ViaContainerVT =
3795 if (ViaVecLen != RequiredVL)
3812 if (EltBitSize - SignBits < 8) {
3816 Source, DAG, Subtarget);
3833 MVT VT =
Op.getSimpleValueType();
3905 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3909 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3925 unsigned NumUndefElts =
3927 unsigned NumDefElts = NumElts - NumUndefElts;
3928 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
3935 for (
unsigned i = 0; i < NumElts; i++) {
3937 if (i < NumElts / 2) {
3944 bool SelectMaskVal = (i < NumElts / 2);
3947 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
3948 MaskVals.
size() == NumElts);
3983 unsigned UndefCount = 0;
3990 LinearBudget -= PerSlideCost;
3993 LinearBudget -= PerSlideCost;
3996 LinearBudget -= PerSlideCost;
3999 if (LinearBudget < 0)
4004 "Illegal type which will result in reserved encoding");
4029 Vec,
Offset, Mask, VL, Policy);
4042 Vec,
Offset, Mask, VL, Policy);
4052 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4053 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4054 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4057 if ((LoC >> 31) == HiC)
4068 (isa<RegisterSDNode>(VL) &&
4069 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4071 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4086 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4087 Hi.getConstantOperandVal(1) == 31)
4106 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4118 bool HasPassthru = Passthru && !Passthru.
isUndef();
4119 if (!HasPassthru && !Passthru)
4127 if (Scalar.getValueType().bitsLE(XLenVT)) {
4134 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4138 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4139 "Unexpected scalar for splat lowering!");
4163 SDValue ExtractedVal = Scalar.getOperand(0);
4168 MVT ExtractedContainerVT = ExtractedVT;
4171 DAG, ExtractedContainerVT, Subtarget);
4173 ExtractedVal, DAG, Subtarget);
4175 if (ExtractedContainerVT.
bitsLE(VT))
4190 if (!Scalar.getValueType().bitsLE(XLenVT))
4193 VT,
DL, DAG, Subtarget);
4201 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4227 if (Src != V2.getOperand(0))
4231 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4236 V2.getConstantOperandVal(1) != Mask.size())
4240 if (Mask[0] != 0 && Mask[0] != 1)
4245 for (
unsigned i = 1; i != Mask.size(); ++i)
4246 if (Mask[i] != Mask[i - 1] + 2)
4264 int Size = Mask.size();
4266 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4272 EvenSrc = StartIndexes[0];
4273 OddSrc = StartIndexes[1];
4276 if (EvenSrc != 0 && OddSrc != 0)
4286 int HalfNumElts = NumElts / 2;
4287 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4303 int Size = Mask.size();
4315 for (
int i = 0; i !=
Size; ++i) {
4321 int StartIdx = i - (M %
Size);
4329 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4332 Rotation = CandidateRotation;
4333 else if (Rotation != CandidateRotation)
4338 int MaskSrc = M <
Size ? 0 : 1;
4343 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4348 TargetSrc = MaskSrc;
4349 else if (TargetSrc != MaskSrc)
4356 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4357 assert((LoSrc >= 0 || HiSrc >= 0) &&
4358 "Failed to find a rotated input vector!");
4373 MVT ContainerVT = VT;
4376 assert(Src.getSimpleValueType().isFixedLengthVector());
4380 MVT SrcContainerVT =
4393 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4400 unsigned Shift = EvenElts ? 0 : EltBits;
4406 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4432 auto findNonEXTRACT_SUBVECTORParent =
4433 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4438 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4439 Offset += Parent.getConstantOperandVal(1);
4440 Parent = Parent.getOperand(0);
4442 return std::make_pair(Parent,
Offset);
4445 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4446 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4455 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4456 if (NewMask[i] == -1)
4459 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4460 NewMask[i] = NewMask[i] + V1IndexOffset;
4464 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4470 if (NewMask[0] <= 0)
4474 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4475 if (NewMask[i - 1] + 1 != NewMask[i])
4479 MVT SrcVT = Src.getSimpleValueType();
4510 int NumSubElts,
Index;
4515 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4516 SDValue InPlace = OpsSwapped ? V2 : V1;
4517 SDValue ToInsert = OpsSwapped ? V1 : V2;
4527 if (NumSubElts +
Index >= (
int)NumElts)
4541 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4553 bool OpsSwapped =
false;
4554 if (!isa<BuildVectorSDNode>(V1)) {
4555 if (!isa<BuildVectorSDNode>(V2))
4560 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4568 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4569 for (
unsigned i = S; i != E; ++i)
4570 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4576 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4577 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4580 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4582 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4587 auto OpCode = IsVSlidedown ?
4592 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4595 Splat, TrueMask, VL);
4606 MVT VecContainerVT = VecVT;
4623 MVT WideContainerVT = WideVT;
4629 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4642 }
else if (Subtarget.hasStdExtZvbb()) {
4647 OffsetVec, Passthru, Mask, VL);
4650 Interleaved, EvenV, Passthru, Mask, VL);
4658 Interleaved, OffsetVec, Passthru, Mask, VL);
4666 OddV, Passthru, Mask, VL);
4672 OddV, AllOnesVec, Passthru, Mask, VL);
4680 Interleaved, OddsMul, Passthru, Mask, VL);
4687 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4733 if (ViaEltSize > NumElts)
4742 if (ViaEltSize > NumElts)
4748 if (ViaEltSize > NumElts)
4757 MVT &RotateVT,
unsigned &RotateAmt) {
4763 unsigned NumSubElts;
4765 NumElts, NumSubElts, RotateAmt))
4768 NumElts / NumSubElts);
4830 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4833 OutMasks(VRegsPerSrc, {-1, {}});
4838 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4839 int DstVecIdx = DstIdx / ElemsPerVReg;
4840 int DstSubIdx = DstIdx % ElemsPerVReg;
4841 int SrcIdx = Mask[DstIdx];
4842 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4844 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4845 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4846 if (OutMasks[DstVecIdx].first == -1)
4847 OutMasks[DstVecIdx].first = SrcVecIdx;
4848 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4854 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4855 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4869 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4870 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4871 if (SrcVecIdx == -1)
4873 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4880 unsigned InsertIdx = DstVecIdx * NumOpElts;
4893 MVT VT =
Op.getSimpleValueType();
4908 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
4932 V.getOperand(0).getSimpleValueType().getVectorNumElements();
4933 V = V.getOperand(
Offset / OpElements);
4939 auto *Ld = cast<LoadSDNode>(V);
4949 SDValue Ops[] = {Ld->getChain(),
4967 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
4968 Ld->getPointerInfo().getWithOffset(
Offset),
4969 Ld->getOriginalAlign(),
4973 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
4974 Ld->getOriginalAlign(),
4975 Ld->getMemOperand()->getFlags());
4986 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
4989 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5011 if (Subtarget.hasStdExtZvkb())
5022 LoV = LoSrc == 0 ? V1 : V2;
5026 HiV = HiSrc == 0 ? V1 : V2;
5032 unsigned InvRotate = NumElts - Rotation;
5042 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5062 int EvenSrc, OddSrc;
5067 int Size = Mask.size();
5069 assert(EvenSrc >= 0 &&
"Undef source?");
5070 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5074 assert(OddSrc >= 0 &&
"Undef source?");
5075 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5084 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5093 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5122 MVT IndexContainerVT =
5127 for (
int MaskIndex : Mask) {
5128 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5137 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5151 int MaskIndex = MaskIdx.value();
5152 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
5157 for (
int MaskIndex : Mask) {
5158 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
5165 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5178 for (
int MaskIndex : Mask) {
5179 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5181 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5182 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5184 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5189 std::swap(ShuffleMaskLHS, ShuffleMaskRHS);
5192 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5228RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5230 MVT VT =
Op.getSimpleValueType();
5234 MVT ContainerVT = VT;
5237 if (
Op->isVPOpcode()) {
5238 Mask =
Op.getOperand(1);
5242 VL =
Op.getOperand(2);
5248 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5250 FloatEltVT = MVT::f32;
5257 "Expected legal float type!");
5264 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5267 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5272 if (FloatVT.
bitsGT(VT)) {
5273 if (
Op->isVPOpcode())
5274 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5283 if (!
Op->isVPOpcode())
5287 MVT ContainerFloatVT =
5290 Src, Mask, RTZRM, VL);
5297 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5301 if (
Op->isVPOpcode()) {
5310 else if (IntVT.
bitsGT(VT))
5315 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5320 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5321 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5326 unsigned Adjust = ExponentBias + (EltSize - 1);
5328 if (
Op->isVPOpcode())
5338 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5339 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5350 auto *
Load = cast<LoadSDNode>(
Op);
5351 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5354 Load->getMemoryVT(),
5355 *
Load->getMemOperand()))
5359 MVT VT =
Op.getSimpleValueType();
5361 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5362 "Unexpected unaligned RVV load type");
5366 "Expecting equally-sized RVV vector types to be legal");
5368 Load->getPointerInfo(),
Load->getOriginalAlign(),
5369 Load->getMemOperand()->getFlags());
5379 auto *
Store = cast<StoreSDNode>(
Op);
5380 assert(Store &&
Store->getValue().getValueType().isVector() &&
5381 "Expected vector store");
5384 Store->getMemoryVT(),
5385 *
Store->getMemOperand()))
5392 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5393 "Unexpected unaligned RVV store type");
5397 "Expecting equally-sized RVV vector types to be legal");
5398 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5400 Store->getPointerInfo(),
Store->getOriginalAlign(),
5401 Store->getMemOperand()->getFlags());
5406 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5408 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5435 unsigned ShiftAmt, AddOpc;
5452 if (Subtarget.hasStdExtZtso()) {
5475 "Unexpected custom legalisation");
5496 "Unexpected custom legalisation");
5511 "Unexpected custom legalisation");
5512 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5532 "Unexpected custom legalisation");
5548 MVT VT =
Op.getSimpleValueType();
5550 unsigned Check =
Op.getConstantOperandVal(1);
5551 unsigned TDCMask = 0;
5579 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5584 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5586 VL =
Op.getOperand(3);
5589 VL,
Op->getFlags());
5604 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5606 MVT MaskContainerVT =
5609 VL =
Op.getOperand(3);
5614 Mask, VL,
Op->getFlags());
5617 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5622 DAG.
getUNDEF(ContainerVT), Mask, VL});
5626 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5630 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5634 DAG.
getUNDEF(ContainerVT), Mask, VL});
5650 MVT VT =
Op.getSimpleValueType();
5677 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5684 MVT ContainerVT = VT;
5692 if (
Op->isVPOpcode()) {
5693 Mask =
Op.getOperand(2);
5697 VL =
Op.getOperand(3);
5705 {X, X, DAG.getCondCode(ISD::SETOEQ),
5706 DAG.getUNDEF(ContainerVT), Mask, VL});
5714 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5715 DAG.getUNDEF(ContainerVT), Mask, VL});
5725 DAG.
getUNDEF(ContainerVT), Mask, VL);
5733#define OP_CASE(NODE) \
5735 return RISCVISD::NODE##_VL;
5736#define VP_CASE(NODE) \
5737 case ISD::VP_##NODE: \
5738 return RISCVISD::NODE##_VL;
5740 switch (
Op.getOpcode()) {
5816 case ISD::VP_CTLZ_ZERO_UNDEF:
5819 case ISD::VP_CTTZ_ZERO_UNDEF:
5828 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5833 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5838 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5841 case ISD::VP_SELECT:
5850 case ISD::VP_SIGN_EXTEND:
5852 case ISD::VP_ZERO_EXTEND:
5854 case ISD::VP_FP_TO_SINT:
5856 case ISD::VP_FP_TO_UINT:
5859 case ISD::VP_FMINNUM:
5862 case ISD::VP_FMAXNUM:
5867 case ISD::VP_LLRINT:
5879 "not a RISC-V target specific op");
5885 "adding target specific op should update this function");
5905 "not a RISC-V target specific op");
5911 "adding target specific op should update this function");
5930 if (!
Op.getOperand(j).getValueType().isVector()) {
5931 LoOperands[j] =
Op.getOperand(j);
5932 HiOperands[j] =
Op.getOperand(j);
5935 std::tie(LoOperands[j], HiOperands[j]) =
5940 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5942 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5957 std::tie(LoOperands[j], HiOperands[j]) =
5961 if (!
Op.getOperand(j).getValueType().isVector()) {
5962 LoOperands[j] =
Op.getOperand(j);
5963 HiOperands[j] =
Op.getOperand(j);
5966 std::tie(LoOperands[j], HiOperands[j]) =
5971 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5973 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5983 auto [EVLLo, EVLHi] =
5984 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
5988 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
5990 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6008 if (!
Op.getOperand(j).getValueType().isVector()) {
6009 LoOperands[j] =
Op.getOperand(j);
6010 HiOperands[j] =
Op.getOperand(j);
6013 std::tie(LoOperands[j], HiOperands[j]) =
6018 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6021 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6030 switch (
Op.getOpcode()) {
6036 return lowerGlobalAddress(
Op, DAG);
6038 return lowerBlockAddress(
Op, DAG);
6040 return lowerConstantPool(
Op, DAG);
6042 return lowerJumpTable(
Op, DAG);
6044 return lowerGlobalTLSAddress(
Op, DAG);
6048 return lowerSELECT(
Op, DAG);
6050 return lowerBRCOND(
Op, DAG);
6052 return lowerVASTART(
Op, DAG);
6054 return lowerFRAMEADDR(
Op, DAG);
6056 return lowerRETURNADDR(
Op, DAG);
6063 return lowerShiftLeftParts(
Op, DAG);
6065 return lowerShiftRightParts(
Op, DAG,
true);
6067 return lowerShiftRightParts(
Op, DAG,
false);
6070 if (
Op.getValueType().isFixedLengthVector()) {
6071 assert(Subtarget.hasStdExtZvkb());
6072 return lowerToScalableOp(
Op, DAG);
6074 assert(Subtarget.hasVendorXTHeadBb() &&
6075 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6076 "Unexpected custom legalization");
6078 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6083 EVT VT =
Op.getValueType();
6087 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6093 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6094 Subtarget.hasStdExtZfbfmin()) {
6099 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6106 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6123 "Unexpected types");
6157 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6159 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6161 return LowerINTRINSIC_VOID(
Op, DAG);
6163 return LowerIS_FPCLASS(
Op, DAG);
6165 MVT VT =
Op.getSimpleValueType();
6167 assert(Subtarget.hasStdExtZvbb());
6168 return lowerToScalableOp(
Op, DAG);
6171 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6179 if (!
Op.getSimpleValueType().isVector())
6181 return lowerVectorTruncLike(
Op, DAG);
6184 if (
Op.getOperand(0).getValueType().isVector() &&
6185 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6186 return lowerVectorMaskExt(
Op, DAG, 1);
6189 if (
Op.getOperand(0).getValueType().isVector() &&
6190 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6191 return lowerVectorMaskExt(
Op, DAG, -1);
6194 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6196 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6198 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6200 MVT VT =
Op.getSimpleValueType();
6208 MVT ContainerVT = VT;
6214 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6221 MVT VT =
Op.getSimpleValueType();
6241 }
else if ((Val % 8) == 0) {
6257 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6258 Op.getOperand(1).getValueType() == MVT::i32) {
6270 if (
Op.getValueType() == MVT::nxv32f16 &&
6277 EVT VT =
Op.getValueType();
6280 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6282 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6288 if (!
Op.getValueType().isVector())
6290 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6294 EVT VT =
Op.getValueType();
6297 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6299 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6307 if (!
Op.getValueType().isVector())
6309 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6313 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6316 if (
Op.getValueType().isVector() &&
6317 Op.getValueType().getScalarType() == MVT::f16 &&
6320 if (
Op.getValueType() == MVT::nxv32f16)
6335 Op1.getValueType().isVector() &&
6336 Op1.getValueType().getScalarType() == MVT::f16 &&
6339 if (Op1.getValueType() == MVT::nxv32f16)
6344 Op1.getValueType().getVectorElementCount());
6347 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6357 MVT VT =
Op.getSimpleValueType();
6361 bool IsStrict =
Op->isStrictFPOpcode();
6362 SDValue Src =
Op.getOperand(0 + IsStrict);
6364 MVT SrcVT = Src.getSimpleValueType();
6369 "Unexpected vector element types");
6373 if (EltSize > (2 * SrcEltSize)) {
6385 Op.getOperand(0), Ext);
6389 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6394 auto [FExt, Chain] =
6396 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6403 if (SrcEltSize > (2 * EltSize)) {
6406 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6411 Op.getOperand(0), Src);
6426 Op.getOperand(0), Src);
6440 unsigned RVVOpc = 0;
6441 switch (
Op.getOpcode()) {
6473 "Expected same element count");
6480 Op.getOperand(0), Src, Mask, VL);
6484 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6499 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6506 MVT VT =
Op.getSimpleValueType();
6528 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6544 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6564 return lowerVECREDUCE(
Op, DAG);
6568 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6569 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6570 return lowerVECREDUCE(
Op, DAG);
6577 return lowerFPVECREDUCE(
Op, DAG);
6578 case ISD::VP_REDUCE_ADD:
6579 case ISD::VP_REDUCE_UMAX:
6580 case ISD::VP_REDUCE_SMAX:
6581 case ISD::VP_REDUCE_UMIN:
6582 case ISD::VP_REDUCE_SMIN:
6583 case ISD::VP_REDUCE_FADD:
6584 case ISD::VP_REDUCE_SEQ_FADD:
6585 case ISD::VP_REDUCE_FMIN:
6586 case ISD::VP_REDUCE_FMAX:
6587 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6591 return lowerVPREDUCE(
Op, DAG);
6592 case ISD::VP_REDUCE_AND:
6593 case ISD::VP_REDUCE_OR:
6594 case ISD::VP_REDUCE_XOR:
6595 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6596 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6597 return lowerVPREDUCE(
Op, DAG);
6601 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6604 return lowerINSERT_SUBVECTOR(
Op, DAG);
6606 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6608 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6610 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6612 return lowerSTEP_VECTOR(
Op, DAG);
6614 return lowerVECTOR_REVERSE(
Op, DAG);
6616 return lowerVECTOR_SPLICE(
Op, DAG);
6620 if (
Op.getValueType().getScalarType() == MVT::f16 &&
6623 if (
Op.getValueType() == MVT::nxv32f16)
6635 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6636 return lowerVectorMaskSplat(
Op, DAG);
6644 MVT VT =
Op.getSimpleValueType();
6645 MVT ContainerVT = VT;
6663 Op->ops().take_front(HalfNumOps));
6665 Op->ops().drop_front(HalfNumOps));
6669 unsigned NumOpElts =
6670 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6673 SDValue SubVec = OpIdx.value();
6684 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6686 if (
Op.getValueType().isFixedLengthVector())
6687 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6690 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6692 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6693 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6697 return lowerMaskedLoad(
Op, DAG);
6700 return lowerMaskedStore(
Op, DAG);
6709 EVT VT =
Op.getValueType();
6720 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6722 MVT VT =
Op.getSimpleValueType();
6727 "Unexpected CondCode");
6735 if (isa<ConstantSDNode>(
RHS)) {
6736 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6737 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6756 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6761 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6777 return lowerToScalableOp(
Op, DAG);
6781 if (
Op.getSimpleValueType().isFixedLengthVector())
6782 return lowerToScalableOp(
Op, DAG);
6784 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6785 "Unexpected custom legalisation");
6797 if (
Op.getValueType() == MVT::nxv32f16 &&
6808 return lowerToScalableOp(
Op, DAG);
6811 if (!
Op.getValueType().isVector())
6813 return lowerToScalableOp(
Op, DAG);
6816 if (!
Op.getValueType().isVector())
6818 return lowerToScalableOp(
Op, DAG);
6822 EVT VT =
Op->getValueType(0);
6837 return lowerABS(
Op, DAG);
6842 if (Subtarget.hasStdExtZvbb())
6843 return lowerToScalableOp(
Op, DAG);
6845 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6847 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6849 if (
Op.getValueType() == MVT::nxv32f16 &&
6853 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6860 if (
Op.getValueType() == MVT::nxv32f16 &&
6864 return lowerToScalableOp(
Op, DAG);
6867 return lowerVectorStrictFSetcc(
Op, DAG);
6877 case ISD::VP_GATHER:
6878 return lowerMaskedGather(
Op, DAG);
6880 case ISD::VP_SCATTER:
6881 return lowerMaskedScatter(
Op, DAG);
6883 return lowerGET_ROUNDING(
Op, DAG);
6885 return lowerSET_ROUNDING(
Op, DAG);
6887 return lowerEH_DWARF_CFA(
Op, DAG);
6888 case ISD::VP_SELECT:
6897 case ISD::VP_UADDSAT:
6898 case ISD::VP_USUBSAT:
6899 case ISD::VP_SADDSAT:
6900 case ISD::VP_SSUBSAT:
6902 case ISD::VP_LLRINT:
6903 return lowerVPOp(
Op, DAG);
6907 return lowerLogicVPOp(
Op, DAG);
6916 case ISD::VP_FMINNUM:
6917 case ISD::VP_FMAXNUM:
6918 case ISD::VP_FCOPYSIGN:
6919 if (
Op.getValueType() == MVT::nxv32f16 &&
6927 return lowerVPOp(
Op, DAG);
6928 case ISD::VP_IS_FPCLASS:
6929 return LowerIS_FPCLASS(
Op, DAG);
6930 case ISD::VP_SIGN_EXTEND:
6931 case ISD::VP_ZERO_EXTEND:
6932 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6933 return lowerVPExtMaskOp(
Op, DAG);
6934 return lowerVPOp(
Op, DAG);
6935 case ISD::VP_TRUNCATE:
6936 return lowerVectorTruncLike(
Op, DAG);
6937 case ISD::VP_FP_EXTEND:
6938 case ISD::VP_FP_ROUND:
6939 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6940 case ISD::VP_SINT_TO_FP:
6941 case ISD::VP_UINT_TO_FP:
6942 if (
Op.getValueType().isVector() &&
6943 Op.getValueType().getScalarType() == MVT::f16 &&
6946 if (
Op.getValueType() == MVT::nxv32f16)
6958 case ISD::VP_FP_TO_SINT:
6959 case ISD::VP_FP_TO_UINT:
6961 Op1.getValueType().isVector() &&
6962 Op1.getValueType().getScalarType() == MVT::f16 &&
6965 if (Op1.getValueType() == MVT::nxv32f16)
6970 Op1.getValueType().getVectorElementCount());
6974 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
6976 return lowerVPFPIntConvOp(
Op, DAG);
6978 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6982 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6983 return lowerVPSetCCMaskOp(
Op, DAG);
6989 case ISD::VP_BITREVERSE:
6991 return lowerVPOp(
Op, DAG);
6993 case ISD::VP_CTLZ_ZERO_UNDEF:
6994 if (Subtarget.hasStdExtZvbb())
6995 return lowerVPOp(
Op, DAG);
6996 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6998 case ISD::VP_CTTZ_ZERO_UNDEF:
6999 if (Subtarget.hasStdExtZvbb())
7000 return lowerVPOp(
Op, DAG);
7001 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7003 return lowerVPOp(
Op, DAG);
7004 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7005 return lowerVPStridedLoad(
Op, DAG);
7006 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7007 return lowerVPStridedStore(
Op, DAG);
7009 case ISD::VP_FFLOOR:
7011 case ISD::VP_FNEARBYINT:
7012 case ISD::VP_FROUND:
7013 case ISD::VP_FROUNDEVEN:
7014 case ISD::VP_FROUNDTOZERO:
7015 if (
Op.getValueType() == MVT::nxv32f16 &&
7020 case ISD::VP_FMAXIMUM:
7021 case ISD::VP_FMINIMUM:
7022 if (
Op.getValueType() == MVT::nxv32f16 &&
7027 case ISD::EXPERIMENTAL_VP_SPLICE:
7028 return lowerVPSpliceExperimental(
Op, DAG);
7029 case ISD::EXPERIMENTAL_VP_REVERSE:
7030 return lowerVPReverseExperimental(
Op, DAG);
7048 N->getOffset(), Flags);
7056template <
class NodeTy>
7058 bool IsLocal,
bool IsExternWeak)
const {
7068 if (IsLocal && !Subtarget.allowTaggedGlobals())
7130 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7139 return getAddr(
N, DAG);
7146 return getAddr(
N, DAG);
7153 return getAddr(
N, DAG);
7158 bool UseGOT)
const {
7222 Args.push_back(Entry);
7255 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7269 Addr = getStaticTLSAddr(
N, DAG,
false);
7272 Addr = getStaticTLSAddr(
N, DAG,
true);
7277 : getDynamicTLSAddr(
N, DAG);
7294 if (
LHS == LHS2 &&
RHS == RHS2) {
7299 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7307 return std::nullopt;
7315 MVT VT =
N->getSimpleValueType(0);
7345 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7348 if (~TrueVal == FalseVal) {
7388 if (Subtarget.hasShortForwardBranchOpt())
7391 unsigned SelOpNo = 0;
7401 unsigned ConstSelOpNo = 1;
7402 unsigned OtherSelOpNo = 2;
7403 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7408 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7409 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7413 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7414 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7420 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7422 std::swap(NewConstOps[0], NewConstOps[1]);
7434 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7436 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7439 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7440 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7449 MVT VT =
Op.getSimpleValueType();
7463 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7491 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7495 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7497 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7498 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7500 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7505 DL, VT, LHSVal, CondV);
7521 if (
Op.hasOneUse()) {
7522 unsigned UseOpc =
Op->use_begin()->getOpcode();
7528 return lowerSELECT(NewSel, DAG);
7556 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7577 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7581 if (TrueVal - 1 == FalseVal)
7583 if (TrueVal + 1 == FalseVal)
7590 RHS == TrueV && LHS == FalseV) {
7607 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7633 LHS, RHS, TargetCC,
Op.getOperand(2));
7651 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7663 int XLenInBytes = Subtarget.
getXLen() / 8;
7665 EVT VT =
Op.getValueType();
7668 unsigned Depth =
Op.getConstantOperandVal(0);
7670 int Offset = -(XLenInBytes * 2);
7686 int XLenInBytes = Subtarget.
getXLen() / 8;
7691 EVT VT =
Op.getValueType();
7693 unsigned Depth =
Op.getConstantOperandVal(0);
7695 int Off = -XLenInBytes;
7696 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7715 EVT VT =
Lo.getValueType();
7754 EVT VT =
Lo.getValueType();
7805 MVT VT =
Op.getSimpleValueType();
7831 MVT VecVT =
Op.getSimpleValueType();
7833 "Unexpected SPLAT_VECTOR_PARTS lowering");
7839 MVT ContainerVT = VecVT;
7859 int64_t ExtTrueVal)
const {
7861 MVT VecVT =
Op.getSimpleValueType();
7864 assert(Src.getValueType().isVector() &&
7865 Src.getValueType().getVectorElementType() == MVT::i1);
7886 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7888 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
7891 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
7896SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
7898 MVT ExtVT =
Op.getSimpleValueType();
7902 MVT VT =
Op.getOperand(0).getSimpleValueType();
7928 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7930 EVT MaskVT =
Op.getValueType();
7933 "Unexpected type for vector mask lowering");
7935 MVT VecVT = Src.getSimpleValueType();
7939 VL =
Op.getOperand(2);
7942 MVT ContainerVT = VecVT;
7948 MVT MaskContainerVT =
7955 std::tie(Mask, VL) =
7963 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
7965 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7969 DAG.
getUNDEF(ContainerVT), Mask, VL);
7972 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
7980 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7983 MVT VT =
Op.getSimpleValueType();
7985 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
7989 return lowerVectorMaskTruncLike(
Op, DAG);
7997 MVT SrcVT = Src.getSimpleValueType();
8002 "Unexpected vector truncate lowering");
8004 MVT ContainerVT = SrcVT;
8008 VL =
Op.getOperand(2);
8021 std::tie(Mask, VL) =
8032 }
while (SrcEltVT != DstEltVT);
8041RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8046 MVT VT =
Op.getSimpleValueType();
8047 MVT SrcVT = Src.getSimpleValueType();
8048 MVT ContainerVT = VT;
8069 Chain, Src, Mask, VL);
8070 Chain = Src.getValue(1);
8077 Chain, Src, Mask, VL);
8088RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8091 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8098 MVT VT =
Op.getSimpleValueType();
8100 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8103 MVT SrcVT = Src.getSimpleValueType();
8110 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8113 MVT ContainerVT = VT;
8117 VL =
Op.getOperand(2);
8131 std::tie(Mask, VL) =
8137 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8143 unsigned InterConvOpc =
8148 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8150 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8161static std::optional<MVT>
8167 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8169 if (MaxIdx < MinVLMAX)
8171 else if (MaxIdx < MinVLMAX * 2)
8173 else if (MaxIdx < MinVLMAX * 4)
8178 return std::nullopt;
8191 MVT VecVT =
Op.getSimpleValueType();
8205 MVT ContainerVT = VecVT;
8214 MVT OrigContainerVT = ContainerVT;
8217 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8218 const unsigned OrigIdx = IdxC->getZExtValue();
8221 DL, DAG, Subtarget)) {
8222 ContainerVT = *ShrunkVT;
8231 VLEN && ContainerVT.
bitsGT(M1VT)) {
8234 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8235 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8236 unsigned ExtractIdx =
8255 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8256 const auto *CVal = cast<ConstantSDNode>(Val);
8257 if (isInt<32>(CVal->getSExtValue())) {
8258 IsLegalInsert =
true;
8267 if (IsLegalInsert) {
8273 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8289 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8290 MVT I32ContainerVT =
8301 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8306 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8308 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8313 ValInVec, AlignedIdx);
8323 DAG.
getUNDEF(I32ContainerVT), ValLo,
8324 I32Mask, InsertI64VL);
8326 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8327 I32Mask, InsertI64VL);
8329 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8342 Idx, Mask, InsertVL, Policy);
8346 Slideup, AlignedIdx);
8361 EVT EltVT =
Op.getValueType();
8368 MVT ContainerVT = VecVT;
8384 unsigned WidenVecLen;
8387 unsigned MaxEEW = Subtarget.
getELen();
8392 "the number of elements should be power of 2");
8396 ExtractBitIdx =
Idx;
8398 WideEltVT = LargestEltVT;
8401 ExtractElementIdx = DAG.
getNode(
8412 Vec, ExtractElementIdx);
8428 MVT ContainerVT = VecVT;
8439 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8440 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8442 unsigned OrigIdx = IdxC->getZExtValue();
8445 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8446 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8447 unsigned ExtractIdx =
8457 std::optional<uint64_t> MaxIdx;
8460 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8461 MaxIdx = IdxC->getZExtValue();
8463 if (
auto SmallerVT =
8465 ContainerVT = *SmallerVT;
8512 "Unexpected opcode");
8519 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8524 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8541 if (OpVT.
bitsLT(XLenVT)) {
8548 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8559 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8562 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8573 case Intrinsic::riscv_vslide1up:
8574 case Intrinsic::riscv_vslide1down:
8575 case Intrinsic::riscv_vslide1up_mask:
8576 case Intrinsic::riscv_vslide1down_mask: {
8579 bool IsMasked = NumOps == 7;
8585 std::tie(ScalarLo, ScalarHi) =
8593 if (isa<ConstantSDNode>(AVL)) {
8594 const auto [MinVLMAX, MaxVLMAX] =
8598 if (AVLInt <= MinVLMAX) {
8600 }
else if (AVLInt >= 2 * MaxVLMAX) {
8607 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8641 if (IntNo == Intrinsic::riscv_vslide1up ||
8642 IntNo == Intrinsic::riscv_vslide1up_mask) {
8644 ScalarHi, I32Mask, I32VL);
8646 ScalarLo, I32Mask, I32VL);
8649 ScalarLo, I32Mask, I32VL);
8651 ScalarHi, I32Mask, I32VL);
8700 const unsigned ElementWidth = 8;
8705 [[maybe_unused]]
unsigned MinVF =
8708 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8712 bool Fractional = VF < LMul1VF;
8713 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8734 MVT ContainerVT = OpVT;
8761 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8765 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8782 if (OpVT.
bitsLT(XLenVT)) {
8785 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8798 EVT ValType = V.getValueType();
8799 if (ValType.isVector() && ValType.isFloatingPoint()) {
8802 ValType.getVectorElementCount());
8805 if (ValType.isFixedLengthVector()) {
8807 DAG, V.getSimpleValueType(), Subtarget);
8823 unsigned IntNo =
Op.getConstantOperandVal(0);
8830 case Intrinsic::thread_pointer: {
8834 case Intrinsic::riscv_orc_b:
8835 case Intrinsic::riscv_brev8:
8836 case Intrinsic::riscv_sha256sig0:
8837 case Intrinsic::riscv_sha256sig1:
8838 case Intrinsic::riscv_sha256sum0:
8839 case Intrinsic::riscv_sha256sum1:
8840 case Intrinsic::riscv_sm3p0:
8841 case Intrinsic::riscv_sm3p1: {
8861 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8863 case Intrinsic::riscv_sm4ks:
8864 case Intrinsic::riscv_sm4ed: {
8874 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
8878 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
8881 case Intrinsic::riscv_zip:
8882 case Intrinsic::riscv_unzip: {
8885 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8887 case Intrinsic::riscv_mopr: {
8900 case Intrinsic::riscv_moprr: {
8912 Op.getOperand(2),
Op.getOperand(3));
8914 case Intrinsic::riscv_clmul:
8925 case Intrinsic::riscv_clmulh:
8926 case Intrinsic::riscv_clmulr: {
8944 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
8946 case Intrinsic::experimental_get_vector_length:
8948 case Intrinsic::experimental_cttz_elts:
8950 case Intrinsic::riscv_vmv_x_s: {
8954 case Intrinsic::riscv_vfmv_f_s:
8957 case Intrinsic::riscv_vmv_v_x:
8959 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
8961 case Intrinsic::riscv_vfmv_v_f:
8963 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
8964 case Intrinsic::riscv_vmv_s_x: {
8967 if (
Scalar.getValueType().bitsLE(XLenVT)) {
8970 Op.getOperand(1), Scalar,
Op.getOperand(3));
8973 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
8990 MVT VT =
Op.getSimpleValueType();
8995 if (
Op.getOperand(1).isUndef())
9011 case Intrinsic::riscv_vfmv_s_f:
9013 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9015 case Intrinsic::riscv_vaesdf_vv:
9016 case Intrinsic::riscv_vaesdf_vs:
9017 case Intrinsic::riscv_vaesdm_vv:
9018 case Intrinsic::riscv_vaesdm_vs:
9019 case Intrinsic::riscv_vaesef_vv:
9020 case Intrinsic::riscv_vaesef_vs:
9021 case Intrinsic::riscv_vaesem_vv:
9022 case Intrinsic::riscv_vaesem_vs:
9023 case Intrinsic::riscv_vaeskf1:
9024 case Intrinsic::riscv_vaeskf2:
9025 case Intrinsic::riscv_vaesz_vs:
9026 case Intrinsic::riscv_vsm4k:
9027 case Intrinsic::riscv_vsm4r_vv:
9028 case Intrinsic::riscv_vsm4r_vs: {
9029 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9030 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9031 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9036 case Intrinsic::riscv_vsm3c:
9037 case Intrinsic::riscv_vsm3me: {
9038 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9039 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9044 case Intrinsic::riscv_vsha2ch:
9045 case Intrinsic::riscv_vsha2cl:
9046 case Intrinsic::riscv_vsha2ms: {
9047 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9048 !Subtarget.hasStdExtZvknhb())
9050 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9051 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9052 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9056 case Intrinsic::riscv_sf_vc_v_x:
9057 case Intrinsic::riscv_sf_vc_v_i:
9058 case Intrinsic::riscv_sf_vc_v_xv:
9059 case Intrinsic::riscv_sf_vc_v_iv:
9060 case Intrinsic::riscv_sf_vc_v_vv:
9061 case Intrinsic::riscv_sf_vc_v_fv:
9062 case Intrinsic::riscv_sf_vc_v_xvv:
9063 case Intrinsic::riscv_sf_vc_v_ivv:
9064 case Intrinsic::riscv_sf_vc_v_vvv:
9065 case Intrinsic::riscv_sf_vc_v_fvv:
9066 case Intrinsic::riscv_sf_vc_v_xvw:
9067 case Intrinsic::riscv_sf_vc_v_ivw:
9068 case Intrinsic::riscv_sf_vc_v_vvw:
9069 case Intrinsic::riscv_sf_vc_v_fvw: {
9070 MVT VT =
Op.getSimpleValueType();
9107 MVT VT =
Op.getSimpleValueType();
9111 if (VT.isFloatingPoint()) {
9116 if (VT.isFixedLengthVector())
9126 if (VT.isFixedLengthVector())
9128 if (VT.isFloatingPoint())
9147 unsigned IntNo =
Op.getConstantOperandVal(1);
9151 case Intrinsic::riscv_masked_strided_load: {
9160 MVT VT =
Op->getSimpleValueType(0);
9161 MVT ContainerVT = VT;
9174 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9186 ScalarVT,
Load->getMemOperand());
9192 Load->getMemOperand());
9197 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9202 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9204 Ops.push_back(PassThru);
9206 Ops.push_back(Stride);
9208 Ops.push_back(Mask);
9213 Ops.push_back(Policy);
9219 Load->getMemoryVT(),
Load->getMemOperand());
9220 Chain =
Result.getValue(1);
9226 case Intrinsic::riscv_seg2_load:
9227 case Intrinsic::riscv_seg3_load:
9228 case Intrinsic::riscv_seg4_load:
9229 case Intrinsic::riscv_seg5_load:
9230 case Intrinsic::riscv_seg6_load:
9231 case Intrinsic::riscv_seg7_load:
9232 case Intrinsic::riscv_seg8_load: {
9235 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9236 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9237 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9238 Intrinsic::riscv_vlseg8};
9239 unsigned NF =
Op->getNumValues() - 1;
9240 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9242 MVT VT =
Op->getSimpleValueType(0);
9248 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9250 ContainerVTs.push_back(MVT::Other);
9258 Load->getMemoryVT(),
Load->getMemOperand());
9260 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9266 case Intrinsic::riscv_sf_vc_v_x_se:
9268 case Intrinsic::riscv_sf_vc_v_i_se:
9270 case Intrinsic::riscv_sf_vc_v_xv_se:
9272 case Intrinsic::riscv_sf_vc_v_iv_se:
9274 case Intrinsic::riscv_sf_vc_v_vv_se:
9276 case Intrinsic::riscv_sf_vc_v_fv_se:
9278 case Intrinsic::riscv_sf_vc_v_xvv_se:
9280 case Intrinsic::riscv_sf_vc_v_ivv_se:
9282 case Intrinsic::riscv_sf_vc_v_vvv_se:
9284 case Intrinsic::riscv_sf_vc_v_fvv_se:
9286 case Intrinsic::riscv_sf_vc_v_xvw_se:
9288 case Intrinsic::riscv_sf_vc_v_ivw_se:
9290 case Intrinsic::riscv_sf_vc_v_vvw_se:
9292 case Intrinsic::riscv_sf_vc_v_fvw_se:
9301 unsigned IntNo =
Op.getConstantOperandVal(1);
9305 case Intrinsic::riscv_masked_strided_store: {
9316 MVT ContainerVT = VT;
9330 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9333 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9343 Ops,
Store->getMemoryVT(),
9344 Store->getMemOperand());
9346 case Intrinsic::riscv_seg2_store:
9347 case Intrinsic::riscv_seg3_store:
9348 case Intrinsic::riscv_seg4_store:
9349 case Intrinsic::riscv_seg5_store:
9350 case Intrinsic::riscv_seg6_store:
9351 case Intrinsic::riscv_seg7_store:
9352 case Intrinsic::riscv_seg8_store: {
9355 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9356 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9357 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9358 Intrinsic::riscv_vsseg8};
9361 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9363 MVT VT =
Op->getOperand(2).getSimpleValueType();
9371 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9373 for (
unsigned i = 0; i < NF; i++)
9375 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9380 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9382 case Intrinsic::riscv_sf_vc_xv_se:
9384 case Intrinsic::riscv_sf_vc_iv_se:
9386 case Intrinsic::riscv_sf_vc_vv_se:
9388 case Intrinsic::riscv_sf_vc_fv_se:
9390 case Intrinsic::riscv_sf_vc_xvv_se:
9392 case Intrinsic::riscv_sf_vc_ivv_se:
9394 case Intrinsic::riscv_sf_vc_vvv_se:
9396 case Intrinsic::riscv_sf_vc_fvv_se:
9398 case Intrinsic::riscv_sf_vc_xvw_se:
9400 case Intrinsic::riscv_sf_vc_ivw_se:
9402 case Intrinsic::riscv_sf_vc_vvw_se:
9404 case Intrinsic::riscv_sf_vc_fvw_se:
9412 switch (ISDOpcode) {
9415 case ISD::VP_REDUCE_ADD:
9418 case ISD::VP_REDUCE_UMAX:
9421 case ISD::VP_REDUCE_SMAX:
9424 case ISD::VP_REDUCE_UMIN:
9427 case ISD::VP_REDUCE_SMIN:
9430 case ISD::VP_REDUCE_AND:
9433 case ISD::VP_REDUCE_OR:
9436 case ISD::VP_REDUCE_XOR:
9439 case ISD::VP_REDUCE_FADD:
9441 case ISD::VP_REDUCE_SEQ_FADD:
9443 case ISD::VP_REDUCE_FMAX:
9445 case ISD::VP_REDUCE_FMIN:
9455 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9460 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9461 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9462 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9463 "Unexpected reduction lowering");
9467 MVT ContainerVT = VecVT;
9476 VL =
Op.getOperand(3);
9478 std::tie(Mask, VL) =
9486 switch (
Op.getOpcode()) {
9490 case ISD::VP_REDUCE_AND: {
9500 case ISD::VP_REDUCE_OR:
9507 case ISD::VP_REDUCE_XOR: {
9530 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9534 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9535 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9536 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9537 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9553 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9557 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9560 if (M1VT != InnerVT)
9566 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9585 VecEVT =
Lo.getValueType();
9598 MVT ContainerVT = VecVT;
9618 Mask, VL,
DL, DAG, Subtarget);
9624static std::tuple<unsigned, SDValue, SDValue>
9628 auto Flags =
Op->getFlags();
9629 unsigned Opcode =
Op.getOpcode();
9653 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9661 MVT VecEltVT =
Op.getSimpleValueType();
9665 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9669 MVT ContainerVT = VecVT;
9675 MVT ResVT =
Op.getSimpleValueType();
9678 VL,
DL, DAG, Subtarget);
9683 if (
Op->getFlags().hasNoNaNs())
9689 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9690 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9696 DL, ResVT, NoNaNs, Res,
9723 Vec, Mask, VL,
DL, DAG, Subtarget);
9735 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9744 (OrigIdx != 0 || !Vec.
isUndef())) {
9747 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9750 "Unexpected mask vector lowering");
9783 MVT ContainerVT = VecVT;
9819 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9820 SlideupAmt, Mask, VL, Policy);
9828 unsigned SubRegIdx, RemIdx;
9829 std::tie(SubRegIdx, RemIdx) =
9831 VecVT, SubVecVT, OrigIdx,
TRI);
9850 if (RemIdx == 0 && (!IsSubVecPartReg || Vec.
isUndef()))
9858 MVT InterSubVT = VecVT;
9860 unsigned AlignedIdx = OrigIdx - RemIdx;
9896 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
9897 SlideupAmt, Mask, VL, Policy);
9902 if (VecVT.
bitsGT(InterSubVT))
9908 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
9914 MVT SubVecVT =
Op.getSimpleValueType();
9919 unsigned OrigIdx =
Op.getConstantOperandVal(1);
9930 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9933 "Unexpected mask vector lowering");
9972 MVT ContainerVT = VecVT;
9982 ContainerVT = *ShrunkVT;
9996 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10008 MVT ContainerSubVecVT = SubVecVT;
10012 unsigned SubRegIdx;
10022 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10023 SubRegIdx = Decompose.first;
10025 (OrigIdx % Vscale));
10029 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10030 SubRegIdx = Decompose.first;
10053 MVT InterSubVT = VecVT;
10057 assert(SubRegIdx != RISCV::NoSubRegister);
10071 Vec, SlidedownAmt, Mask, VL);
10080 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10087 MVT VT =
N.getSimpleValueType();
10091 assert(
Op.getSimpleValueType() == VT &&
10092 "Operands and result must be same type");
10096 unsigned NumVals =
N->getNumValues();
10099 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10102 for (
unsigned I = 0;
I < NumVals;
I++) {
10108 if (TruncVals.
size() > 1)
10110 return TruncVals.
front();
10116 MVT VecVT =
Op.getSimpleValueType();
10119 "vector_interleave on non-scalable vector!");
10130 EVT SplitVT = Op0Lo.getValueType();
10133 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10135 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10149 Op.getOperand(0),
Op.getOperand(1));
10176 Concat, EvenIdx, Passthru, Mask, VL);
10178 Concat, OddIdx, Passthru, Mask, VL);
10192 MVT VecVT =
Op.getSimpleValueType();
10195 "vector_interleave on non-scalable vector!");
10208 EVT SplitVT = Op0Lo.getValueType();
10211 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10213 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10235 Op.getOperand(0),
Op.getOperand(1));
10283 MVT VT =
Op.getSimpleValueType();
10288 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10289 if (StepValImm != 1) {
10298 VL, VT,
DL, DAG, Subtarget);
10313 MVT VecVT =
Op.getSimpleValueType();
10323 unsigned MaxVLMAX =
10333 if (MaxVLMAX > 256 && EltSize == 8) {
10380 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10390 MVT VecVT =
Op.getSimpleValueType();
10394 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10395 SDValue DownOffset, UpOffset;
10396 if (ImmValue >= 0) {
10412 DownOffset, TrueMask, UpOffset);
10413 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10419RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10422 auto *
Load = cast<LoadSDNode>(
Op);
10425 Load->getMemoryVT(),
10426 *
Load->getMemOperand()) &&
10427 "Expecting a correctly-aligned load");
10429 MVT VT =
Op.getSimpleValueType();
10435 const auto [MinVLMAX, MaxVLMAX] =
10438 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10452 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10461 Load->getMemoryVT(),
Load->getMemOperand());
10468RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10471 auto *
Store = cast<StoreSDNode>(
Op);
10474 Store->getMemoryVT(),
10475 *
Store->getMemOperand()) &&
10476 "Expecting a correctly-aligned store");
10498 const auto [MinVLMAX, MaxVLMAX] =
10501 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10513 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10516 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10517 Store->getMemoryVT(),
Store->getMemOperand());
10523 MVT VT =
Op.getSimpleValueType();
10525 const auto *MemSD = cast<MemSDNode>(
Op);
10526 EVT MemVT = MemSD->getMemoryVT();
10528 SDValue Chain = MemSD->getChain();
10532 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10533 Mask = VPLoad->getMask();
10535 VL = VPLoad->getVectorLength();
10537 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10538 Mask = MLoad->getMask();
10539 PassThru = MLoad->getPassThru();
10546 MVT ContainerVT = VT;
10560 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10577 Chain =
Result.getValue(1);
10589 const auto *MemSD = cast<MemSDNode>(
Op);
10590 EVT MemVT = MemSD->getMemoryVT();
10592 SDValue Chain = MemSD->getChain();
10596 bool IsCompressingStore =
false;
10597 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10598 Val = VPStore->getValue();
10599 Mask = VPStore->getMask();
10600 VL = VPStore->getVectorLength();
10602 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10603 Val = MStore->getValue();
10604 Mask = MStore->getMask();
10605 IsCompressingStore = MStore->isCompressingStore();
10614 MVT ContainerVT = VT;
10619 if (!IsUnmasked || IsCompressingStore) {
10628 if (IsCompressingStore) {
10631 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10638 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10647 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10651RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10653 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10656 MVT VT =
Op.getSimpleValueType();
10670 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10677 unsigned Opc =
Op.getOpcode();
10684 MVT VT =
Op.getSimpleValueType();
10717 MVT ContainerInVT = InVT;
10736 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10740 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10748 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10753 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10766 MVT VT =
Op.getSimpleValueType();
10770 "Unexpected type for ISD::ABS");
10772 MVT ContainerVT = VT;
10779 if (
Op->getOpcode() == ISD::VP_ABS) {
10780 Mask =
Op->getOperand(1);
10784 VL =
Op->getOperand(2);
10792 DAG.
getUNDEF(ContainerVT), Mask, VL);
10794 DAG.
getUNDEF(ContainerVT), Mask, VL);
10801SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
10804 MVT VT =
Op.getSimpleValueType();
10808 "Can only handle COPYSIGN with matching types.");
10817 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
10822SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
10824 MVT VT =
Op.getSimpleValueType();
10827 MVT I1ContainerVT =
10841 Op2, DAG.
getUNDEF(ContainerVT), VL);
10852 MVT VT =
Op.getSimpleValueType();
10857 for (
const SDValue &V :
Op->op_values()) {
10858 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10861 if (!
V.getValueType().isVector()) {
10867 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
10868 "Only fixed length vectors are supported!");
10882 if (
Op->isStrictFPOpcode()) {
10891 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
10905 MVT VT =
Op.getSimpleValueType();
10908 MVT ContainerVT = VT;
10914 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10920 if (*MaskIdx == OpIdx.index())
10924 if (
Op.getOpcode() == ISD::VP_MERGE) {
10928 assert(
Op.getOpcode() == ISD::VP_SELECT);
10935 if (!
V.getValueType().isFixedLengthVector()) {
10940 MVT OpVT =
V.getSimpleValueType();
10942 assert(useRVVForFixedLengthVectorVT(OpVT) &&
10943 "Only fixed length vectors are supported!");
10948 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
10958 MVT VT =
Op.getSimpleValueType();
10964 MVT ContainerVT = VT;
10974 DAG.
getUNDEF(ContainerVT), Zero, VL);
10977 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
10979 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
10982 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
10991 MVT VT =
Op.getSimpleValueType();
10995 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
10999 MVT ContainerVT = VT;
11009 switch (Condition) {
11077 MVT DstVT =
Op.getSimpleValueType();
11078 MVT SrcVT = Src.getSimpleValueType();
11091 if (DstEltSize >= SrcEltSize) {
11100 if (SrcEltSize == 1) {
11111 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11112 }
else if (DstEltSize > (2 * SrcEltSize)) {
11116 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11122 "Wrong input/output vector types");
11125 if (DstEltSize > (2 * SrcEltSize)) {
11141 MVT InterimFVT = DstVT;
11142 if (SrcEltSize > (2 * DstEltSize)) {
11143 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11150 if (InterimFVT != DstVT) {
11156 "Wrong input/output vector types");
11160 if (DstEltSize == 1) {
11163 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11173 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11183 while (InterimIVT != DstVT) {
11195 MVT VT =
Op.getSimpleValueType();
11202RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11214 MVT VT =
Op.getSimpleValueType();
11215 MVT ContainerVT = VT;
11228 if (IsMaskVector) {
11239 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11248 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11251 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11252 SDValue DownOffset, UpOffset;
11253 if (ImmValue >= 0) {
11267 Op1, DownOffset, Mask, UpOffset);
11271 if (IsMaskVector) {
11275 {Result, DAG.getConstant(0, DL, ContainerVT),
11276 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11286RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11289 MVT VT =
Op.getSimpleValueType();
11296 MVT ContainerVT = VT;
11304 MVT GatherVT = ContainerVT;
11308 if (IsMaskVector) {
11319 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11325 unsigned MaxVLMAX =
11334 if (MaxVLMAX > 256 && EltSize == 8) {
11362 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11364 if (IsMaskVector) {
11387 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11389 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11391 DAG.
getUNDEF(GatherVT), Mask, EVL);
11393 if (IsMaskVector) {
11408 MVT VT =
Op.getSimpleValueType();
11410 return lowerVPOp(
Op, DAG);
11417 MVT ContainerVT = VT;
11436 MVT VT =
Op.getSimpleValueType();
11437 MVT ContainerVT = VT;
11443 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11449 : Intrinsic::riscv_vlse_mask,
11452 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11453 VPNode->getStride()};
11461 Ops.
push_back(VPNode->getVectorLength());
11469 VPNode->getMemoryVT(), VPNode->getMemOperand());
11483 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11484 SDValue StoreVal = VPNode->getValue();
11486 MVT ContainerVT = VT;
11497 : Intrinsic::riscv_vsse_mask,
11500 VPNode->getBasePtr(), VPNode->getStride()};
11508 Ops.
push_back(VPNode->getVectorLength());
11511 Ops, VPNode->getMemoryVT(),
11512 VPNode->getMemOperand());
11524 MVT VT =
Op.getSimpleValueType();
11526 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11527 EVT MemVT = MemSD->getMemoryVT();
11529 SDValue Chain = MemSD->getChain();
11535 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11536 Index = VPGN->getIndex();
11537 Mask = VPGN->getMask();
11539 VL = VPGN->getVectorLength();
11544 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11545 Index = MGN->getIndex();
11546 Mask = MGN->getMask();
11547 PassThru = MGN->getPassThru();
11551 MVT IndexVT =
Index.getSimpleValueType();
11555 "Unexpected VTs!");
11556 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11559 "Unexpected extending MGATHER/VP_GATHER");
11565 MVT ContainerVT = VT;
11589 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11606 Chain =
Result.getValue(1);
11623 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11624 EVT MemVT = MemSD->getMemoryVT();
11626 SDValue Chain = MemSD->getChain();
11629 [[maybe_unused]]
bool IsTruncatingStore =
false;
11632 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11633 Index = VPSN->getIndex();
11634 Mask = VPSN->getMask();
11635 Val = VPSN->getValue();
11636 VL = VPSN->getVectorLength();
11638 IsTruncatingStore =
false;
11641 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11642 Index = MSN->getIndex();
11643 Mask = MSN->getMask();
11644 Val = MSN->getValue();
11645 IsTruncatingStore = MSN->isTruncatingStore();
11649 MVT IndexVT =
Index.getSimpleValueType();
11653 "Unexpected VTs!");
11654 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11657 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11663 MVT ContainerVT = VT;
11687 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11697 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11706 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11714 static const int Table =
11738 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11744 static const unsigned Table =
11767 bool isRISCV64 = Subtarget.
is64Bit();
11831 switch (
N->getOpcode()) {
11833 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
11839 "Unexpected custom legalisation");
11840 bool IsStrict =
N->isStrictFPOpcode();
11843 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
11861 Opc,
DL, VTs, Chain, Op0,
11895 std::tie(Result, Chain) =
11896 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
11924 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
11935 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
11936 "has custom type legalization on riscv32");
11938 SDValue LoCounter, HiCounter;
11942 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
11944 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
11947 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
11949 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
11953 N->getOperand(0), LoCounter, HiCounter);
11977 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
11978 unsigned XLen = Subtarget.
getXLen();
11981 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
11989 if (LHSIsU == RHSIsU)
12006 if (RHSIsU && LHSIsS && !RHSIsS)
12008 else if (LHSIsU && RHSIsS && !LHSIsS)
12018 "Unexpected custom legalisation");
12025 "Unexpected custom legalisation");
12028 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12054 "Unexpected custom legalisation");
12055 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12056 Subtarget.hasVendorXTHeadBb()) &&
12057 "Unexpected custom legalization");
12058 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12059 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12068 "Unexpected custom legalisation");
12082 MVT VT =
N->getSimpleValueType(0);
12083 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12084 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12085 "Unexpected custom legalisation");
12097 if (VT != MVT::i32)
12106 "Unexpected custom legalisation");
12110 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12127 EVT OType =
N->getValueType(1);
12140 "Unexpected custom legalisation");
12157 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12161 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12179 "Unexpected custom legalisation");
12180 if (Subtarget.hasStdExtZbb()) {
12201 "Unexpected custom legalisation");
12207 "Unexpected custom legalisation");
12209 if (Subtarget.hasStdExtZbb()) {
12243 EVT VT =
N->getValueType(0);
12248 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12252 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12253 Subtarget.hasStdExtZfbfmin()) {
12256 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12261 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12263 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12283 MVT VT =
N->getSimpleValueType(0);
12285 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12286 "Unexpected custom legalisation");
12287 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
12313 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12315 "Unexpected EXTRACT_VECTOR_ELT legalization");
12318 MVT ContainerVT = VecVT;
12346 DAG.
getUNDEF(ContainerVT), Mask, VL);
12354 unsigned IntNo =
N->getConstantOperandVal(0);
12358 "Don't know how to custom type legalize this intrinsic!");
12359 case Intrinsic::experimental_get_vector_length: {
12364 case Intrinsic::experimental_cttz_elts: {
12370 case Intrinsic::riscv_orc_b:
12371 case Intrinsic::riscv_brev8:
12372 case Intrinsic::riscv_sha256sig0:
12373 case Intrinsic::riscv_sha256sig1:
12374 case Intrinsic::riscv_sha256sum0:
12375 case Intrinsic::riscv_sha256sum1:
12376 case Intrinsic::riscv_sm3p0:
12377 case Intrinsic::riscv_sm3p1: {
12378 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12398 case Intrinsic::riscv_sm4ks:
12399 case Intrinsic::riscv_sm4ed: {
12407 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12411 case Intrinsic::riscv_mopr: {
12412 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12422 case Intrinsic::riscv_moprr: {
12423 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12435 case Intrinsic::riscv_clmul: {
12436 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12447 case Intrinsic::riscv_clmulh:
12448 case Intrinsic::riscv_clmulr: {
12449 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12477 case Intrinsic::riscv_vmv_x_s: {
12478 EVT VT =
N->getValueType(0);
12480 if (VT.
bitsLT(XLenVT)) {
12489 "Unexpected custom legalization");
12527 case ISD::VP_REDUCE_ADD:
12528 case ISD::VP_REDUCE_AND:
12529 case ISD::VP_REDUCE_OR:
12530 case ISD::VP_REDUCE_XOR:
12531 case ISD::VP_REDUCE_SMAX:
12532 case ISD::VP_REDUCE_UMAX:
12533 case ISD::VP_REDUCE_SMIN:
12534 case ISD::VP_REDUCE_UMIN:
12598 const EVT VT =
N->getValueType(0);
12599 const unsigned Opc =
N->getOpcode();
12606 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12611 "Inconsistent mappings");
12622 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12625 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12640 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12642 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12643 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12647 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12654 if (
LHS.getOpcode() != ReduceOpc)
12669 auto Flags = ReduceVec->
getFlags();
12670 Flags.intersectWith(
N->getFlags());
12671 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12681 auto BinOpToRVVReduce = [](
unsigned Opc) {
12710 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12713 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12716 unsigned Opc =
N->getOpcode();
12717 unsigned ReduceIdx;
12718 if (IsReduction(
N->getOperand(0), Opc))
12720 else if (IsReduction(
N->getOperand(1), Opc))
12726 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12729 SDValue Extract =
N->getOperand(ReduceIdx);
12761 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
12788 if (!Subtarget.hasStdExtZba())
12792 EVT VT =
N->getValueType(0);
12804 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12805 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
12808 int64_t C0 = N0C->getSExtValue();
12809 int64_t C1 = N1C->getSExtValue();
12810 if (C0 <= 0 || C1 <= 0)
12814 int64_t Bits = std::min(C0, C1);
12815 int64_t Diff = std::abs(C0 - C1);
12816 if (Diff != 1 && Diff != 2 && Diff != 3)
12844 EVT VT =
N->getValueType(0);
12852 if ((!Subtarget.hasStdExtZicond() &&
12853 !Subtarget.hasVendorXVentanaCondOps()) ||
12875 bool SwapSelectOps;
12881 SwapSelectOps =
false;
12882 NonConstantVal = FalseVal;
12884 SwapSelectOps =
true;
12885 NonConstantVal = TrueVal;
12891 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
12939 EVT VT =
N->getValueType(0);
12947 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12948 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12954 if (!N0C->hasOneUse())
12956 int64_t C0 = N0C->getSExtValue();
12957 int64_t C1 = N1C->getSExtValue();
12959 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
12962 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
12963 !isInt<12>(C0 * (C1 / C0))) {
12966 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
12967 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
12970 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
12971 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
12998 EVT VT =
N->getValueType(0);
13029 unsigned OuterExtend =
13033 OuterExtend,
SDLoc(
N), VT,
13041 EVT VT =
N->getValueType(0);
13086 EVT VT =
N->getValueType(0);
13090 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13096 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13106 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13128 EVT VT =
N->getValueType(0);
13157 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13181 EVT VT =
N->getValueType(0);
13201 EVT VT =
N->getValueType(0);
13287 EVT VT =
N->getValueType(0);
13358 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13363 const APInt &Imm = ConstN00->getAPIntValue();
13364 if ((Imm + 1).isSignedIntN(12))
13381 LHS.getValueType());
13402 EVT VT =
N->getValueType(0);
13414 if (!Subtarget.hasStdExtZba() && !Subtarget.hasVendorXTHeadBa())
13428 for (
uint64_t Divisor : {3, 5, 9}) {
13429 if (MulAmt % Divisor != 0)
13431 uint64_t MulAmt2 = MulAmt / Divisor;
13438 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13454 if (ScaleShift >= 1 && ScaleShift < 4) {
13455 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13469 for (
uint64_t Divisor : {3, 5, 9}) {
13474 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13485 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13487 if (ScaleShift >= 1 && ScaleShift < 4) {
13488 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13519 EVT VT =
N->getValueType(0);
13527 unsigned AddSubOpc;
13533 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13534 AddSubOpc = V->getOpcode();
13536 SDValue Opnd = V->getOperand(1);
13537 MulOper = V->getOperand(0);
13546 if (IsAddSubWith1(N0)) {
13548 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13551 if (IsAddSubWith1(N1)) {
13553 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13565 if (isIndexTypeSigned(IndexType))
13568 if (!
N->hasOneUse())
13571 EVT VT =
N.getValueType();
13610 EVT SrcVT = Src.getValueType();
13614 NewElen = std::max(NewElen, 8U);
13637 EVT VT =
N->getValueType(0);
13640 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
13644 auto *N1C = dyn_cast<ConstantSDNode>(N1);
13656 if (!isIntEqualitySetCC(
Cond))
13665 const APInt &C1 = N1C->getAPIntValue();
13683 EVT VT =
N->getValueType(0);
13687 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
13689 Src.getOperand(0));
13697struct CombineResult;
13699enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
13726struct NodeExtensionHelper {
13735 bool SupportsFPExt;
13738 bool EnforceOneUse;
13753 return OrigOperand;
13764 unsigned getExtOpc(ExtKind SupportsExt)
const {
13765 switch (SupportsExt) {
13766 case ExtKind::SExt:
13768 case ExtKind::ZExt:
13770 case ExtKind::FPExt:
13781 std::optional<ExtKind> SupportsExt)
const {
13782 if (!SupportsExt.has_value())
13783 return OrigOperand;
13785 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
13789 if (
Source.getValueType() == NarrowVT)
13792 unsigned ExtOpc = getExtOpc(*SupportsExt);
13796 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
13803 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
13821 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
13827 MVT EltVT = SupportsExt == ExtKind::FPExt
13829 :
MVT::getIntegerVT(NarrowSize);
13831 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
13832 "Trying to extend something we can't represent");
13839 static unsigned getSExtOpcode(
unsigned Opcode) {
13862 static unsigned getZExtOpcode(
unsigned Opcode) {
13888 static unsigned getFPExtOpcode(
unsigned Opcode) {
13905 static unsigned getSUOpcode(
unsigned Opcode) {
13907 "SU is only supported for MUL");
13913 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
13933 using CombineToTry = std::function<std::optional<CombineResult>(
13934 SDNode * ,
const NodeExtensionHelper & ,
13939 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
13943 unsigned Opc = OrigOperand.
getOpcode();
13947 "Unexpected Opcode");
13960 unsigned ScalarBits =
Op.getValueSizeInBits();
13963 if (ScalarBits < EltBits)
13969 if (NarrowSize < 8)
13973 SupportsSExt =
true;
13977 SupportsZExt =
true;
13979 EnforceOneUse =
false;
13986 SupportsZExt =
false;
13987 SupportsSExt =
false;
13988 SupportsFPExt =
false;
13989 EnforceOneUse =
true;
13990 unsigned Opc = OrigOperand.
getOpcode();
14012 SupportsZExt =
true;
14015 SupportsSExt =
true;
14018 SupportsFPExt =
true;
14022 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14030 static bool isSupportedRoot(
const SDNode *Root,
14059 Subtarget.hasStdExtZvbb();
14061 return Subtarget.hasStdExtZvbb();
14070 assert(isSupportedRoot(Root, Subtarget) &&
14071 "Trying to build an helper with an "
14072 "unsupported root");
14073 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14089 if (OperandIdx == 1) {
14098 EnforceOneUse =
false;
14103 fillUpExtensionSupport(Root, DAG, Subtarget);
14109 static std::pair<SDValue, SDValue>
14112 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14131 switch (
N->getOpcode()) {
14169struct CombineResult {
14171 unsigned TargetOpcode;
14173 std::optional<ExtKind> LHSExt;
14174 std::optional<ExtKind> RHSExt;
14178 NodeExtensionHelper
LHS;
14180 NodeExtensionHelper
RHS;
14182 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14183 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14184 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14185 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14194 std::tie(Mask, VL) =
14195 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14209 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14210 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14225static std::optional<CombineResult>
14226canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14227 const NodeExtensionHelper &RHS,
14230 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14231 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14232 Root, LHS, {ExtKind::ZExt}, RHS,
14234 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14235 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14236 Root, LHS, {ExtKind::SExt}, RHS,
14238 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14239 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14240 Root, LHS, {ExtKind::FPExt}, RHS,
14242 return std::nullopt;
14251static std::optional<CombineResult>
14252canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14255 return canFoldToVWWithSameExtensionImpl(
14256 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14264static std::optional<CombineResult>
14265canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14268 if (
RHS.SupportsFPExt)
14269 return CombineResult(
14270 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14271 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14278 return CombineResult(
14279 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14280 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14282 return CombineResult(
14283 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14284 LHS, std::nullopt, RHS, {ExtKind::SExt});
14285 return std::nullopt;
14292static std::optional<CombineResult>
14293canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14296 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14304static std::optional<CombineResult>
14305canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14308 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14316static std::optional<CombineResult>
14317canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14320 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14328static std::optional<CombineResult>
14329canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14333 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14334 return std::nullopt;
14335 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14336 Root, LHS, {ExtKind::SExt}, RHS,
14341NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14352 Strategies.
push_back(canFoldToVWWithSameExtension);
14357 Strategies.
push_back(canFoldToVWWithSameExtension);
14362 Strategies.
push_back(canFoldToVWWithSameExtension);
14369 Strategies.
push_back(canFoldToVWWithZEXT);
14374 Strategies.
push_back(canFoldToVWWithSEXT);
14379 Strategies.
push_back(canFoldToVWWithZEXT);
14384 Strategies.
push_back(canFoldToVWWithFPEXT);
14413 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14419 Inserted.insert(
N);
14422 while (!Worklist.
empty()) {
14424 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14427 NodeExtensionHelper
LHS(
N, 0, DAG, Subtarget);
14428 NodeExtensionHelper
RHS(
N, 1, DAG, Subtarget);
14429 auto AppendUsersIfNeeded = [&Worklist,
14430 &Inserted](
const NodeExtensionHelper &
Op) {
14431 if (
Op.needToPromoteOtherUsers()) {
14432 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14433 if (Inserted.insert(TheUse).second)
14445 NodeExtensionHelper::getSupportedFoldings(
N);
14447 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14448 bool Matched =
false;
14449 for (
int Attempt = 0;
14450 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
14453 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14454 FoldingStrategies) {
14455 std::optional<CombineResult> Res =
14456 FoldingStrategy(
N,
LHS,
RHS, DAG, Subtarget);
14463 if (Res->LHSExt.has_value())
14464 AppendUsersIfNeeded(
LHS);
14465 if (Res->RHSExt.has_value())
14466 AppendUsersIfNeeded(
RHS);
14477 SDValue InputRootReplacement;
14484 for (CombineResult Res : CombinesToApply) {
14485 SDValue NewValue = Res.materialize(DAG, Subtarget);
14486 if (!InputRootReplacement) {
14488 "First element is expected to be the current node");
14489 InputRootReplacement = NewValue;
14494 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14498 return InputRootReplacement;
14505 unsigned Opc =
N->getOpcode();
14510 SDValue MergeOp =
N->getOperand(1);
14511 unsigned MergeOpc = MergeOp.
getOpcode();
14522 SDValue Passthru =
N->getOperand(2);
14536 Z = Z.getOperand(1);
14542 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14549 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14578 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14584 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14586 if (MemVT == MVT::i32)
14592 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
14627 if (!Subtarget.hasVendorXTHeadMemPair())
14639 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
14641 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
14642 return {
Ptr->getOperand(0), C1->getZExtValue()};
14646 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
14669 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
14672 if (Base1 != Base2)
14676 bool Valid =
false;
14677 if (MemVT == MVT::i32) {
14679 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
14681 }
else if (MemVT == MVT::i64) {
14683 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
14717 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14725 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14735 EVT VT =
N->getValueType(0);
14738 MVT SrcVT = Src.getSimpleValueType();
14739 MVT SrcContainerVT = SrcVT;
14741 SDValue XVal = Src.getOperand(0);
14768 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14772 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14776 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
14789 if (VT != MVT::i32 && VT != XLenVT)
14819 EVT DstVT =
N->getValueType(0);
14820 if (DstVT != XLenVT)
14826 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14834 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14837 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
14846 if (SatVT == DstVT)
14848 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
14854 Src = Src.getOperand(0);
14875 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
14881 EVT VT =
N->getValueType(0);
14934 unsigned Offset =
N->isTargetStrictFPOpcode();
14941 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
14943 V.getOperand(2) == VL) {
14945 V = V.getOperand(0);
14952 bool NegA = invertIfNegative(
A);
14953 bool NegB = invertIfNegative(
B);
14954 bool NegC = invertIfNegative(
C);
14957 if (!NegA && !NegB && !NegC)
14961 if (
N->isTargetStrictFPOpcode())
14963 {N->getOperand(0), A, B, C, Mask, VL});
14973 if (
N->getValueType(0).isScalableVector() &&
14974 N->getValueType(0).getVectorElementType() == MVT::f32 &&
14981 if (
N->isTargetStrictFPOpcode())
15006 switch (
N->getOpcode()) {
15026 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15027 N->getOperand(2), Mask, VL);
15034 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15037 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15039 uint64_t ShAmt =
N->getConstantOperandVal(1);
15050 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15078 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15091 !isa<ConstantSDNode>(U->getOperand(1)) ||
15092 U->getConstantOperandVal(1) > 32)
15147 if (!
Cond.hasOneUse())
15166 EVT VT =
Cond.getValueType();
15211 LHS =
LHS.getOperand(0);
15221 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15225 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15229 RHS =
LHS.getOperand(1);
15230 LHS =
LHS.getOperand(0);
15239 RHS =
LHS.getOperand(1);
15240 LHS =
LHS.getOperand(0);
15256 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15297 bool Commutative =
true;
15298 unsigned Opc = TrueVal.getOpcode();
15306 Commutative =
false;
15314 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15318 if (FalseVal == TrueVal.getOperand(0))
15320 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15325 EVT VT =
N->getValueType(0);
15327 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15333 assert(IdentityOperand &&
"No identity operand!");
15338 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15339 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15360 CountZeroes =
N->getOperand(2);
15361 ValOnZero =
N->getOperand(1);
15363 CountZeroes =
N->getOperand(1);
15364 ValOnZero =
N->getOperand(2);
15383 if (
Cond->getOperand(0) != CountZeroesArgument)
15399 CountZeroes, BitWidthMinusOne);
15409 EVT VT =
N->getValueType(0);
15410 EVT CondVT =
Cond.getValueType();
15418 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15424 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15445 SDValue TrueVal =
N->getOperand(1);
15446 SDValue FalseVal =
N->getOperand(2);
15461 EVT VT =
N->getValueType(0);
15468 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15483 if (
Op.isUndef()) {
15496 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15500 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15501 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15505 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15533 const unsigned InVecOpcode = InVec->
getOpcode();
15543 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15550 InVecLHS, InValLHS, EltNo);
15552 InVecRHS, InValRHS, EltNo);
15561 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15564 unsigned Elt = IndexC->getZExtValue();
15572 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15575 ConcatOp, InVal, NewIdx);
15579 ConcatOps[ConcatOpIdx] = ConcatOp;
15591 EVT VT =
N->getValueType(0);
15601 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
15603 !
SDValue(BaseLd, 0).hasOneUse())
15606 EVT BaseLdVT = BaseLd->getValueType(0);
15613 auto *Ld = dyn_cast<LoadSDNode>(
Op);
15614 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
15616 Ld->getValueType(0) != BaseLdVT)
15625 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
15627 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
15632 if (BIO1.equalBaseIndex(BIO2, DAG))
15633 return {{BIO2.getOffset() - BIO1.getOffset(),
false}};
15637 SDValue P2 = Ld2->getBasePtr();
15643 return std::nullopt;
15647 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
15652 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
15653 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
15661 unsigned WideScalarBitWidth =
15674 auto [StrideVariant, MustNegateStride] = *BaseDiff;
15675 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
15676 ? std::get<SDValue>(StrideVariant)
15679 if (MustNegateStride)
15692 BaseLd->getBasePtr(), Stride, AllOneMask};
15695 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
15696 ConstStride && ConstStride->getSExtValue() >= 0)
15700 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
15706 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
15710 Ops, WideVecVT, MMO);
15722 if (
N->getValueType(0).isFixedLengthVector())
15725 SDValue Addend =
N->getOperand(0);
15729 SDValue AddMergeOp =
N->getOperand(2);
15734 auto IsVWMulOpc = [](
unsigned Opc) {
15763 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
15764 }(
N, DAG, Subtarget);
15769 if (AddMask != MulMask || AddVL != MulVL)
15774 "Unexpected opcode after VWMACC_VL");
15776 "Unexpected opcode after VWMACC_VL!");
15778 "Unexpected opcode after VWMUL_VL!");
15780 "Unexpected opcode after VWMUL_VL!");
15783 EVT VT =
N->getValueType(0);
15799 const EVT IndexVT =
Index.getValueType();
15803 if (!isIndexTypeSigned(IndexType))
15835 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15838 if (
Index->getOperand(i)->isUndef())
15841 if (
C % ElementSize != 0)
15843 C =
C / ElementSize;
15847 ActiveLanes.
set(
C);
15849 return ActiveLanes.
all();
15867 if (NumElems % 2 != 0)
15871 const unsigned WiderElementSize = ElementSize * 2;
15872 if (WiderElementSize > ST.getELen()/8)
15875 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
15878 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15881 if (
Index->getOperand(i)->isUndef())
15887 if (
C % WiderElementSize != 0)
15892 if (
C !=
Last + ElementSize)
15908 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
15919 switch (
N->getOpcode()) {
15939 APInt V =
C->getValueAPF().bitcastToAPInt();
15974 if (SimplifyDemandedLowBitsHelper(0, 32) ||
15975 SimplifyDemandedLowBitsHelper(1, 5))
15983 if (SimplifyDemandedLowBitsHelper(0, 32))
16000 MVT VT =
N->getSimpleValueType(0);
16009 "Unexpected value type!");
16032 EVT VT =
N->getValueType(0);
16097 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16102 Src.getOperand(0));
16107 Src.getOperand(0), Src.getOperand(1));
16122 auto IsTruncNode = [](
SDValue V) {
16125 SDValue VL = V.getOperand(2);
16126 auto *
C = dyn_cast<ConstantSDNode>(VL);
16128 bool IsVLMAXForVMSET = (
C &&
C->isAllOnes()) ||
16129 (isa<RegisterSDNode>(VL) &&
16130 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16139 while (IsTruncNode(
Op)) {
16140 if (!
Op.hasOneUse())
16142 Op =
Op.getOperand(0);
16174 unsigned Opc =
N->getOpcode();
16189 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16199 N->getValueType(0), Val,
Cond.getOperand(0));
16210 SDValue FalseV =
N->getOperand(4);
16212 EVT VT =
N->getValueType(0);
16215 if (TrueV == FalseV)
16220 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16226 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16227 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16230 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16231 isInt<12>(TrueSImm - FalseSImm)) {
16247 {LHS, RHS, CC, TrueV, FalseV});
16314 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16327 EVT VT =
N->getValueType(0);
16351 const auto *MGN = dyn_cast<MaskedGatherSDNode>(
N);
16352 const EVT VT =
N->getValueType(0);
16354 SDValue ScaleOp = MGN->getScale();
16356 assert(!MGN->isIndexScaled() &&
16357 "Scaled gather/scatter should not be formed");
16362 N->getVTList(), MGN->getMemoryVT(),
DL,
16363 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16364 MGN->getBasePtr(), Index, ScaleOp},
16365 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16369 N->getVTList(), MGN->getMemoryVT(),
DL,
16370 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16371 MGN->getBasePtr(), Index, ScaleOp},
16372 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16378 if (std::optional<VIDSequence> SimpleVID =
16380 SimpleVID && SimpleVID->StepDenominator == 1) {
16381 const int64_t StepNumerator = SimpleVID->StepNumerator;
16382 const int64_t Addend = SimpleVID->Addend;
16389 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16399 {MGN->getChain(), IntID, MGN->getPassThru(), BasePtr,
16400 DAG.
getConstant(StepNumerator,
DL, XLenVT), MGN->getMask()};
16402 Ops, VT, MGN->getMemOperand());
16410 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16412 MGN->getMemoryVT(), MGN->getMemOperand(),
16421 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16423 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16425 EVT IndexVT =
Index.getValueType()
16432 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16434 EltCnt.divideCoefficientBy(2));
16437 EltCnt.divideCoefficientBy(2));
16442 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16451 const auto *MSN = dyn_cast<MaskedScatterSDNode>(
N);
16453 SDValue ScaleOp = MSN->getScale();
16455 assert(!MSN->isIndexScaled() &&
16456 "Scaled gather/scatter should not be formed");
16461 N->getVTList(), MSN->getMemoryVT(),
DL,
16462 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16464 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16468 N->getVTList(), MSN->getMemoryVT(),
DL,
16469 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16471 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16473 EVT VT = MSN->getValue()->getValueType(0);
16475 if (!MSN->isTruncatingStore() &&
16479 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
16480 DAG.
getUNDEF(XLenVT), MSN->getMask(),
16481 MSN->getMemoryVT(), MSN->getMemOperand(),
16486 case ISD::VP_GATHER: {
16487 const auto *VPGN = dyn_cast<VPGatherSDNode>(
N);
16489 SDValue ScaleOp = VPGN->getScale();
16491 assert(!VPGN->isIndexScaled() &&
16492 "Scaled gather/scatter should not be formed");
16497 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16498 ScaleOp, VPGN->getMask(),
16499 VPGN->getVectorLength()},
16500 VPGN->getMemOperand(), IndexType);
16504 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16505 ScaleOp, VPGN->getMask(),
16506 VPGN->getVectorLength()},
16507 VPGN->getMemOperand(), IndexType);
16511 case ISD::VP_SCATTER: {
16512 const auto *VPSN = dyn_cast<VPScatterSDNode>(
N);
16514 SDValue ScaleOp = VPSN->getScale();
16516 assert(!VPSN->isIndexScaled() &&
16517 "Scaled gather/scatter should not be formed");
16522 {VPSN->getChain(), VPSN->getValue(),
16523 VPSN->getBasePtr(), Index, ScaleOp,
16524 VPSN->getMask(), VPSN->getVectorLength()},
16525 VPSN->getMemOperand(), IndexType);
16529 {VPSN->getChain(), VPSN->getValue(),
16530 VPSN->getBasePtr(), Index, ScaleOp,
16531 VPSN->getMask(), VPSN->getVectorLength()},
16532 VPSN->getMemOperand(), IndexType);
16546 EVT VT =
N->getValueType(0);
16549 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
16550 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
16568 EVT VT =
N->getValueType(0);
16572 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
16602 if (
N->getValueType(0).isScalableVector() &&
16603 N->getValueType(0).getVectorElementType() == MVT::f32 &&
16618 auto *Store = cast<StoreSDNode>(
N);
16619 SDValue Chain = Store->getChain();
16620 EVT MemVT = Store->getMemoryVT();
16621 SDValue Val = Store->getValue();
16624 bool IsScalarizable =
16626 Store->isSimple() &&
16656 NewVT, *Store->getMemOperand())) {
16658 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
16659 Store->getPointerInfo(), Store->getOriginalAlign(),
16660 Store->getMemOperand()->getFlags());
16668 if (
auto *L = dyn_cast<LoadSDNode>(Val);
16670 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
16672 L->getMemoryVT() == MemVT) {
16675 NewVT, *Store->getMemOperand()) &&
16677 NewVT, *L->getMemOperand())) {
16679 L->getPointerInfo(), L->getOriginalAlign(),
16680 L->getMemOperand()->getFlags());
16681 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
16682 Store->getPointerInfo(), Store->getOriginalAlign(),
16683 Store->getMemOperand()->getFlags());
16695 MVT VecVT = Src.getSimpleValueType();
16702 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
16705 Store->getMemOperand(), Store->getAddressingMode(),
16706 Store->isTruncatingStore(),
false);
16713 EVT VT =
N->getValueType(0);
16735 const MVT VT =
N->getSimpleValueType(0);
16736 SDValue Passthru =
N->getOperand(0);
16737 SDValue Scalar =
N->getOperand(1);
16746 const MVT VT =
N->getSimpleValueType(0);
16747 SDValue Passthru =
N->getOperand(0);
16748 SDValue Scalar =
N->getOperand(1);
16753 unsigned ScalarSize = Scalar.getValueSizeInBits();
16755 if (ScalarSize > EltWidth && Passthru.
isUndef())
16756 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
16763 (!Const || Const->isZero() ||
16764 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
16774 if (
N->getOperand(0).isUndef() &&
16777 Src.getOperand(0).getValueType().isScalableVector()) {
16778 EVT VT =
N->getValueType(0);
16779 EVT SrcVT = Src.getOperand(0).getValueType();
16783 return Src.getOperand(0);
16789 const MVT VT =
N->getSimpleValueType(0);
16790 SDValue Passthru =
N->getOperand(0);
16791 SDValue Scalar =
N->getOperand(1);
16795 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
16796 return Scalar.getOperand(0);
16805 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
16815 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
16823 MVT VecVT =
N->getOperand(0).getSimpleValueType();
16825 if (M1VT.
bitsLT(VecVT)) {
16836 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
16841 case Intrinsic::riscv_masked_strided_load: {
16842 MVT VT =
N->getSimpleValueType(0);
16843 auto *Load = cast<MemIntrinsicSDNode>(
N);
16844 SDValue PassThru =
N->getOperand(2);
16846 SDValue Stride =
N->getOperand(4);
16852 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16853 StrideC && StrideC->getZExtValue() == ElementSize)
16855 DAG.
getUNDEF(XLenVT), Mask, PassThru,
16856 Load->getMemoryVT(), Load->getMemOperand(),
16860 case Intrinsic::riscv_masked_strided_store: {
16861 auto *Store = cast<MemIntrinsicSDNode>(
N);
16864 SDValue Stride =
N->getOperand(4);
16869 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
16870 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16871 StrideC && StrideC->getZExtValue() == ElementSize)
16874 Value.getValueType(), Store->getMemOperand(),
16878 case Intrinsic::riscv_vcpop:
16879 case Intrinsic::riscv_vcpop_mask:
16880 case Intrinsic::riscv_vfirst:
16881 case Intrinsic::riscv_vfirst_mask: {
16883 if (IntNo == Intrinsic::riscv_vcpop_mask ||
16884 IntNo == Intrinsic::riscv_vfirst_mask)
16885 VL =
N->getOperand(3);
16890 EVT VT =
N->getValueType(0);
16891 if (IntNo == Intrinsic::riscv_vfirst ||
16892 IntNo == Intrinsic::riscv_vfirst_mask)
16901 EVT VT =
N->getValueType(0);
16905 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
16924 EVT XVT,
unsigned KeptBits)
const {
16929 if (XVT != MVT::i32 && XVT != MVT::i64)
16933 if (KeptBits == 32 || KeptBits == 64)
16937 return Subtarget.hasStdExtZbb() &&
16938 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
16946 "Expected shift op");
16957 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
16958 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
16960 const APInt &C1Int = C1->getAPIntValue();
16961 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
16987 if (C1Cost < ShiftedC1Cost)
17001 EVT VT =
Op.getValueType();
17005 unsigned Opcode =
Op.getOpcode();
17013 const APInt &Mask =
C->getAPIntValue();
17022 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17023 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17025 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17026 if (NewMask == Mask)
17031 Op.getOperand(0), NewC);
17044 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17045 if (IsLegalMask(NewMask))
17046 return UseMask(NewMask);
17049 if (VT == MVT::i64) {
17051 if (IsLegalMask(NewMask))
17052 return UseMask(NewMask);
17067 APInt NewMask = ShrunkMask;
17068 if (MinSignedBits <= 12)
17070 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17076 assert(IsLegalMask(NewMask));
17077 return UseMask(NewMask);
17081 static const uint64_t GREVMasks[] = {
17082 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17083 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17085 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17086 unsigned Shift = 1 << Stage;
17087 if (ShAmt & Shift) {
17089 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17101 const APInt &DemandedElts,
17103 unsigned Depth)
const {
17105 unsigned Opc =
Op.getOpcode();
17110 "Should use MaskedValueIsZero if you don't know whether Op"
17111 " is a target node!");
17194 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17197 if (MaxVLenB == MinVLenB)
17214 case Intrinsic::riscv_vsetvli:
17215 case Intrinsic::riscv_vsetvlimax: {
17216 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17217 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17223 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17226 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17227 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17229 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17242 unsigned Depth)
const {
17243 switch (
Op.getOpcode()) {
17249 if (Tmp == 1)
return 1;
17252 return std::min(Tmp, Tmp2);
17264 if (Tmp < 33)
return 1;
17289 unsigned XLen = Subtarget.
getXLen();
17290 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17291 if (EltBits <= XLen)
17292 return XLen - EltBits + 1;
17296 unsigned IntNo =
Op.getConstantOperandVal(1);
17300 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17301 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17302 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17303 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17304 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17305 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17306 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17307 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17308 case Intrinsic::riscv_masked_cmpxchg_i64:
17316 assert(Subtarget.hasStdExtA());
17331 switch (
Op.getOpcode()) {
17337 return !
Op.getValueType().isInteger();
17345 assert(Ld &&
"Unexpected null LoadSDNode");
17353 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17354 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17355 CNode->getOffset() != 0)
17363 auto *CNode = GetSupportedConstantPool(
Ptr);
17364 if (!CNode || CNode->getTargetFlags() != 0)
17367 return CNode->getConstVal();
17375 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17376 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17382 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17385 return CNodeLo->getConstVal();
17390 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17422 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17425 int64_t LoCounter =
MI.getOperand(2).getImm();
17426 int64_t HiCounter =
MI.getOperand(3).getImm();
17436 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17448 MI.eraseFromParent();
17456 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
17464 Register SrcReg =
MI.getOperand(2).getReg();
17484 MI.eraseFromParent();
17491 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
17492 "Unexpected instruction");
17498 Register DstReg =
MI.getOperand(0).getReg();
17521 MI.eraseFromParent();
17526 switch (
MI.getOpcode()) {
17529 case RISCV::Select_GPR_Using_CC_GPR:
17530 case RISCV::Select_FPR16_Using_CC_GPR:
17531 case RISCV::Select_FPR16INX_Using_CC_GPR:
17532 case RISCV::Select_FPR32_Using_CC_GPR:
17533 case RISCV::Select_FPR32INX_Using_CC_GPR:
17534 case RISCV::Select_FPR64_Using_CC_GPR:
17535 case RISCV::Select_FPR64INX_Using_CC_GPR:
17536 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
17542 unsigned RelOpcode,
unsigned EqOpcode,
17545 Register DstReg =
MI.getOperand(0).getReg();
17546 Register Src1Reg =
MI.getOperand(1).getReg();
17547 Register Src2Reg =
MI.getOperand(2).getReg();
17549 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17573 MI.eraseFromParent();
17624 F->insert(It, FirstMBB);
17625 F->insert(It, SecondMBB);
17626 F->insert(It, SinkMBB);
17675 First.eraseFromParent();
17718 SelectDests.
insert(
MI.getOperand(0).getReg());
17722 if (
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR && Next != BB->
end() &&
17723 Next->getOpcode() ==
MI.getOpcode() &&
17724 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
17725 Next->getOperand(5).isKill()) {
17730 SequenceMBBI != E; ++SequenceMBBI) {
17731 if (SequenceMBBI->isDebugInstr())
17734 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
17735 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
17736 SequenceMBBI->getOperand(3).getImm() !=
CC ||
17737 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
17738 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
17740 LastSelectPseudo = &*SequenceMBBI;
17742 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
17745 if (SequenceMBBI->hasUnmodeledSideEffects() ||
17746 SequenceMBBI->mayLoadOrStore() ||
17747 SequenceMBBI->usesCustomInsertionHook())
17750 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
17765 F->insert(
I, IfFalseMBB);
17766 F->insert(
I, TailMBB);
17770 TailMBB->
push_back(DebugInstr->removeFromParent());
17774 TailMBB->
splice(TailMBB->
end(), HeadMBB,
17793 auto SelectMBBI =
MI.getIterator();
17794 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
17795 auto InsertionPoint = TailMBB->
begin();
17796 while (SelectMBBI != SelectEnd) {
17797 auto Next = std::next(SelectMBBI);
17800 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
17801 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
17802 .
addReg(SelectMBBI->getOperand(4).getReg())
17804 .
addReg(SelectMBBI->getOperand(5).getReg())
17819 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
17820 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
17822 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
17823 assert(Masked &&
"Could not find masked instruction for LMUL and SEW pair");
17829 unsigned CVTXOpc) {
17835 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17848 .
add(
MI.getOperand(1))
17849 .
add(
MI.getOperand(2))
17850 .
add(
MI.getOperand(3))
17852 .
add(
MI.getOperand(4))
17853 .
add(
MI.getOperand(5))
17854 .
add(
MI.getOperand(6))
17869 .
add(
MI.getOperand(0))
17870 .
add(
MI.getOperand(1))
17872 .
add(
MI.getOperand(3))
17874 .
add(
MI.getOperand(4))
17875 .
add(
MI.getOperand(5))
17876 .
add(
MI.getOperand(6))
17886 MI.eraseFromParent();
17892 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
17894 switch (
MI.getOpcode()) {
17897 case RISCV::PseudoFROUND_H:
17898 CmpOpc = RISCV::FLT_H;
17899 F2IOpc = RISCV::FCVT_W_H;
17900 I2FOpc = RISCV::FCVT_H_W;
17901 FSGNJOpc = RISCV::FSGNJ_H;
17902 FSGNJXOpc = RISCV::FSGNJX_H;
17903 RC = &RISCV::FPR16RegClass;
17905 case RISCV::PseudoFROUND_H_INX:
17906 CmpOpc = RISCV::FLT_H_INX;
17907 F2IOpc = RISCV::FCVT_W_H_INX;
17908 I2FOpc = RISCV::FCVT_H_W_INX;
17909 FSGNJOpc = RISCV::FSGNJ_H_INX;
17910 FSGNJXOpc = RISCV::FSGNJX_H_INX;
17911 RC = &RISCV::GPRF16RegClass;
17913 case RISCV::PseudoFROUND_S:
17914 CmpOpc = RISCV::FLT_S;
17915 F2IOpc = RISCV::FCVT_W_S;
17916 I2FOpc = RISCV::FCVT_S_W;
17917 FSGNJOpc = RISCV::FSGNJ_S;
17918 FSGNJXOpc = RISCV::FSGNJX_S;
17919 RC = &RISCV::FPR32RegClass;
17921 case RISCV::PseudoFROUND_S_INX:
17922 CmpOpc = RISCV::FLT_S_INX;
17923 F2IOpc = RISCV::FCVT_W_S_INX;
17924 I2FOpc = RISCV::FCVT_S_W_INX;
17925 FSGNJOpc = RISCV::FSGNJ_S_INX;
17926 FSGNJXOpc = RISCV::FSGNJX_S_INX;
17927 RC = &RISCV::GPRF32RegClass;
17929 case RISCV::PseudoFROUND_D:
17931 CmpOpc = RISCV::FLT_D;
17932 F2IOpc = RISCV::FCVT_L_D;
17933 I2FOpc = RISCV::FCVT_D_L;
17934 FSGNJOpc = RISCV::FSGNJ_D;
17935 FSGNJXOpc = RISCV::FSGNJX_D;
17936 RC = &RISCV::FPR64RegClass;
17938 case RISCV::PseudoFROUND_D_INX:
17940 CmpOpc = RISCV::FLT_D_INX;
17941 F2IOpc = RISCV::FCVT_L_D_INX;
17942 I2FOpc = RISCV::FCVT_D_L_INX;
17943 FSGNJOpc = RISCV::FSGNJ_D_INX;
17944 FSGNJXOpc = RISCV::FSGNJX_D_INX;
17945 RC = &RISCV::GPRRegClass;
17957 F->insert(
I, CvtMBB);
17958 F->insert(
I, DoneMBB);
17969 Register DstReg =
MI.getOperand(0).getReg();
17970 Register SrcReg =
MI.getOperand(1).getReg();
17971 Register MaxReg =
MI.getOperand(2).getReg();
17972 int64_t FRM =
MI.getOperand(3).getImm();
17977 Register FabsReg =
MRI.createVirtualRegister(RC);
17981 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17996 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18018 MI.eraseFromParent();
18025 switch (
MI.getOpcode()) {
18028 case RISCV::ReadCounterWide:
18030 "ReadCounterWide is only to be used on riscv32");
18032 case RISCV::Select_GPR_Using_CC_GPR:
18033 case RISCV::Select_FPR16_Using_CC_GPR:
18034 case RISCV::Select_FPR16INX_Using_CC_GPR:
18035 case RISCV::Select_FPR32_Using_CC_GPR:
18036 case RISCV::Select_FPR32INX_Using_CC_GPR:
18037 case RISCV::Select_FPR64_Using_CC_GPR:
18038 case RISCV::Select_FPR64INX_Using_CC_GPR:
18039 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18041 case RISCV::BuildPairF64Pseudo:
18043 case RISCV::SplitF64Pseudo:
18045 case RISCV::PseudoQuietFLE_H:
18047 case RISCV::PseudoQuietFLE_H_INX:
18048 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18049 case RISCV::PseudoQuietFLT_H:
18051 case RISCV::PseudoQuietFLT_H_INX:
18052 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18053 case RISCV::PseudoQuietFLE_S:
18055 case RISCV::PseudoQuietFLE_S_INX:
18056 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18057 case RISCV::PseudoQuietFLT_S:
18059 case RISCV::PseudoQuietFLT_S_INX:
18060 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18061 case RISCV::PseudoQuietFLE_D:
18063 case RISCV::PseudoQuietFLE_D_INX:
18064 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18065 case RISCV::PseudoQuietFLE_D_IN32X:
18068 case RISCV::PseudoQuietFLT_D:
18070 case RISCV::PseudoQuietFLT_D_INX:
18071 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18072 case RISCV::PseudoQuietFLT_D_IN32X:
18076 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18078 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18080 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18082 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18084 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18086 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18088 case RISCV::PseudoFROUND_H:
18089 case RISCV::PseudoFROUND_H_INX:
18090 case RISCV::PseudoFROUND_S:
18091 case RISCV::PseudoFROUND_S_INX:
18092 case RISCV::PseudoFROUND_D:
18093 case RISCV::PseudoFROUND_D_INX:
18094 case RISCV::PseudoFROUND_D_IN32X:
18096 case TargetOpcode::STATEPOINT:
18102 MI.addOperand(*
MI.getMF(),
18108 case TargetOpcode::STACKMAP:
18109 case TargetOpcode::PATCHPOINT:
18112 "supported on 64-bit targets");
18130 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18162 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18163 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18166 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18167 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18170 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18171 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18175 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18176 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18177 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18179 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18180 RISCV::V20M2, RISCV::V22M2};
18188 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18189 RISCV::X13, RISCV::X14, RISCV::X15,
18190 RISCV::X16, RISCV::X17};
18192 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18193 RISCV::X13, RISCV::X14, RISCV::X15};
18204 static const MCPhysReg FastCCIGPRs[] = {
18205 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
18206 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
18207 RISCV::X29, RISCV::X30, RISCV::X31};
18210 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18211 RISCV::X13, RISCV::X14, RISCV::X15,
18226 unsigned XLenInBytes = XLen / 8;
18239 Align StackAlign(XLenInBytes);
18240 if (!
EABI || XLen != 32)
18272 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18273 assert(XLen == 32 || XLen == 64);
18274 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18278 if (ArgFlags.
isNest()) {
18279 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18287 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18292 bool UseGPRForF16_F32 =
true;
18295 bool UseGPRForF64 =
true;
18307 UseGPRForF16_F32 = !IsFixed;
18311 UseGPRForF16_F32 = !IsFixed;
18312 UseGPRForF64 = !IsFixed;
18318 UseGPRForF16_F32 =
true;
18319 UseGPRForF64 =
true;
18326 if (UseGPRForF16_F32 &&
18327 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18330 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18347 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18349 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18353 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18362 "PendingLocs and PendingArgFlags out of sync");
18366 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18367 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18415 PendingLocs.
size() <= 2) {
18416 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18421 PendingLocs.
clear();
18422 PendingArgFlags.
clear();
18424 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18430 unsigned StoreSizeBytes = XLen / 8;
18433 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18435 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18437 else if (ValVT == MVT::f64 && !UseGPRForF64)
18473 if (!PendingLocs.
empty()) {
18475 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
18477 for (
auto &It : PendingLocs) {
18479 It.convertToReg(Reg);
18484 PendingLocs.clear();
18485 PendingArgFlags.
clear();
18489 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
18491 "Expected an XLenVT or vector types at this stage");
18509template <
typename ArgTy>
18511 for (
const auto &ArgIdx :
enumerate(Args)) {
18512 MVT ArgVT = ArgIdx.value().VT;
18514 return ArgIdx.index();
18516 return std::nullopt;
18519void RISCVTargetLowering::analyzeInputArgs(
18522 RISCVCCAssignFn Fn)
const {
18523 unsigned NumArgs =
Ins.size();
18536 for (
unsigned i = 0; i != NumArgs; ++i) {
18540 Type *ArgTy =
nullptr;
18542 ArgTy = FType->getReturnType();
18543 else if (Ins[i].isOrigArg())
18544 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
18548 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
18550 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
18557void RISCVTargetLowering::analyzeOutputArgs(
18560 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
18561 unsigned NumArgs = Outs.
size();
18571 for (
unsigned i = 0; i != NumArgs; i++) {
18572 MVT ArgVT = Outs[i].VT;
18574 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
18578 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
18580 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
18634 if (In.isOrigArg()) {
18639 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
18640 (
BitWidth < 32 && In.Flags.isZExt())) {
18669 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
18715 ExtType,
DL, LocVT, Chain, FIN,
18732 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18745 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18755 unsigned ValNo,
MVT ValVT,
MVT LocVT,
18758 bool IsFixed,
bool IsRet,
Type *OrigTy,
18761 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18770 if (LocVT == MVT::f16 &&
18771 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
18773 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
18774 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
18775 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
18776 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
18777 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
18783 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18785 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
18786 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
18787 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
18788 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
18789 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18795 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18797 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
18798 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
18799 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
18800 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
18801 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18808 if ((LocVT == MVT::f16 &&
18809 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
18810 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18811 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
18812 Subtarget.hasStdExtZdinx())) {
18819 if (LocVT == MVT::f16) {
18825 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
18831 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
18839 if (AllocatedVReg) {
18874 if (ArgFlags.
isNest()) {
18876 "Attribute 'nest' is not supported in GHC calling convention");
18880 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
18881 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
18883 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18895 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18898 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
18899 RISCV::F18_F, RISCV::F19_F,
18900 RISCV::F20_F, RISCV::F21_F};
18901 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18907 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18910 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
18911 RISCV::F24_D, RISCV::F25_D,
18912 RISCV::F26_D, RISCV::F27_D};
18913 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18919 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18920 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
18940 switch (CallConv) {
18950 if (Subtarget.hasStdExtE())
18954 "(Zdinx/D) instruction set extensions");
18958 if (Func.hasFnAttribute(
"interrupt")) {
18959 if (!Func.arg_empty())
18961 "Functions with the interrupt attribute cannot have arguments!");
18966 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
18968 "Function interrupt attribute argument not supported!");
18973 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
18975 std::vector<SDValue> OutChains;
18984 analyzeInputArgs(MF, CCInfo, Ins,
false,
18988 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19009 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19010 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19012 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19014 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19043 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19048 if (VarArgsSaveSize == 0) {
19052 int VaArgOffset = -VarArgsSaveSize;
19060 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19061 VarArgsSaveSize += XLenInBytes;
19068 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19073 Chain,
DL, ArgValue, FIN,
19075 OutChains.push_back(Store);
19089 if (!OutChains.empty()) {
19090 OutChains.push_back(Chain);
19100bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19104 auto CalleeCC = CLI.CallConv;
19105 auto &Outs = CLI.Outs;
19107 auto CallerCC = Caller.getCallingConv();
19114 if (Caller.hasFnAttribute(
"interrupt"))
19129 for (
auto &VA : ArgLocs)
19135 auto IsCallerStructRet = Caller.hasStructRetAttr();
19136 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19137 if (IsCallerStructRet || IsCalleeStructRet)
19142 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19143 if (CalleeCC != CallerCC) {
19144 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19145 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19152 for (
auto &Arg : Outs)
19153 if (Arg.Flags.isByVal())
19188 if (Subtarget.hasStdExtE())
19192 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19198 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19204 "site marked musttail");
19211 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19213 if (!Flags.isByVal())
19217 unsigned Size = Flags.getByValSize();
19218 Align Alignment = Flags.getNonZeroByValAlign();
19225 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19239 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19242 SDValue ArgValue = OutVals[OutIdx];
19262 if (!StackPtr.getNode())
19273 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19291 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19292 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19298 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19299 SDValue PartValue = OutVals[OutIdx + 1];
19300 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19312 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19314 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19316 for (
const auto &Part : Parts) {
19317 SDValue PartValue = Part.first;
19318 SDValue PartOffset = Part.second;
19325 ArgValue = SpillSlot;
19331 if (Flags.isByVal())
19332 ArgValue = ByValArgs[j++];
19339 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19340 "for passing parameters");
19343 if (!StackPtr.getNode())
19356 if (!MemOpChains.
empty())
19362 for (
auto &Reg : RegsToPass) {
19363 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19370 validateCCReservedRegs(RegsToPass, MF);
19375 "Return address register required, but has been reserved."});
19394 for (
auto &Reg : RegsToPass)
19400 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19401 assert(Mask &&
"Missing call preserved mask for calling convention");
19410 "Unexpected CFI type for a direct call");
19440 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19441 auto &VA = RVLocs[i];
19449 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19450 assert(VA.needsCustom());
19475 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19476 MVT VT = Outs[i].VT;
19480 ArgFlags, CCInfo,
true,
true,
19481 nullptr, *
this, Dispatcher))
19513 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19514 SDValue Val = OutVals[OutIdx];
19523 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19527 Register RegHi = RVLocs[++i].getLocReg();
19533 "Return value register required, but has been reserved."});
19549 "Return value register required, but has been reserved."});
19571 if (Func.hasFnAttribute(
"interrupt")) {
19572 if (!Func.getReturnType()->isVoidTy())
19574 "Functions with the interrupt attribute must have void return type!");
19580 if (Kind ==
"supervisor")
19586 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19589void RISCVTargetLowering::validateCCReservedRegs(
19590 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19599 F,
"Argument register required, but has been reserved."});
19605 if (
N->getNumValues() != 1)
19607 if (!
N->hasNUsesOfValue(1, 0))
19610 SDNode *Copy = *
N->use_begin();
19624 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
19628 bool HasRet =
false;
19629 for (
SDNode *Node : Copy->uses()) {
19637 Chain = Copy->getOperand(0);
19646#define NODE_NAME_CASE(NODE) \
19647 case RISCVISD::NODE: \
19648 return "RISCVISD::" #NODE;
19901#undef NODE_NAME_CASE
19908 if (Constraint.
size() == 1) {
19909 switch (Constraint[0]) {
19925 if (Constraint ==
"vr" || Constraint ==
"vm")
19931std::pair<unsigned, const TargetRegisterClass *>
19937 if (Constraint.
size() == 1) {
19938 switch (Constraint[0]) {
19943 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
19944 return std::make_pair(0U, &RISCV::GPRF16RegClass);
19945 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
19946 return std::make_pair(0U, &RISCV::GPRF32RegClass);
19947 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
19948 return std::make_pair(0U, &RISCV::GPRPairRegClass);
19949 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
19951 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
19952 return std::make_pair(0U, &RISCV::FPR16RegClass);
19953 if (Subtarget.hasStdExtF() && VT == MVT::f32)
19954 return std::make_pair(0U, &RISCV::FPR32RegClass);
19955 if (Subtarget.hasStdExtD() && VT == MVT::f64)
19956 return std::make_pair(0U, &RISCV::FPR64RegClass);
19961 }
else if (Constraint ==
"vr") {
19962 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
19963 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
19965 return std::make_pair(0U, RC);
19967 }
else if (Constraint ==
"vm") {
19968 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
19969 return std::make_pair(0U, &RISCV::VMV0RegClass);
19977 .
Case(
"{zero}", RISCV::X0)
19978 .
Case(
"{ra}", RISCV::X1)
19979 .
Case(
"{sp}", RISCV::X2)
19980 .
Case(
"{gp}", RISCV::X3)
19981 .
Case(
"{tp}", RISCV::X4)
19982 .
Case(
"{t0}", RISCV::X5)
19983 .
Case(
"{t1}", RISCV::X6)
19984 .
Case(
"{t2}", RISCV::X7)
19985 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
19986 .
Case(
"{s1}", RISCV::X9)
19987 .
Case(
"{a0}", RISCV::X10)
19988 .
Case(
"{a1}", RISCV::X11)
19989 .
Case(
"{a2}", RISCV::X12)
19990 .
Case(
"{a3}", RISCV::X13)
19991 .
Case(
"{a4}", RISCV::X14)
19992 .
Case(
"{a5}", RISCV::X15)
19993 .
Case(
"{a6}", RISCV::X16)
19994 .
Case(
"{a7}", RISCV::X17)
19995 .
Case(
"{s2}", RISCV::X18)
19996 .
Case(
"{s3}", RISCV::X19)
19997 .
Case(
"{s4}", RISCV::X20)
19998 .
Case(
"{s5}", RISCV::X21)
19999 .
Case(
"{s6}", RISCV::X22)
20000 .
Case(
"{s7}", RISCV::X23)
20001 .
Case(
"{s8}", RISCV::X24)
20002 .
Case(
"{s9}", RISCV::X25)
20003 .
Case(
"{s10}", RISCV::X26)
20004 .
Case(
"{s11}", RISCV::X27)
20005 .
Case(
"{t3}", RISCV::X28)
20006 .
Case(
"{t4}", RISCV::X29)
20007 .
Case(
"{t5}", RISCV::X30)
20008 .
Case(
"{t6}", RISCV::X31)
20010 if (XRegFromAlias != RISCV::NoRegister)
20011 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20020 if (Subtarget.hasStdExtF()) {
20022 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20023 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20024 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20025 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20026 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20027 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20028 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20029 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20030 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20031 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20032 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20033 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20034 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20035 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20036 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20037 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20038 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20039 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20040 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20041 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20042 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20043 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20044 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20045 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20046 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20047 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20048 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20049 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20050 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20051 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20052 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20053 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20055 if (FReg != RISCV::NoRegister) {
20056 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20057 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20058 unsigned RegNo = FReg - RISCV::F0_F;
20059 unsigned DReg = RISCV::F0_D + RegNo;
20060 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20062 if (VT == MVT::f32 || VT == MVT::Other)
20063 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20064 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20065 unsigned RegNo = FReg - RISCV::F0_F;
20066 unsigned HReg = RISCV::F0_H + RegNo;
20067 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20074 .
Case(
"{v0}", RISCV::V0)
20075 .
Case(
"{v1}", RISCV::V1)
20076 .
Case(
"{v2}", RISCV::V2)
20077 .
Case(
"{v3}", RISCV::V3)
20078 .
Case(
"{v4}", RISCV::V4)
20079 .
Case(
"{v5}", RISCV::V5)
20080 .
Case(
"{v6}", RISCV::V6)
20081 .
Case(
"{v7}", RISCV::V7)
20082 .
Case(
"{v8}", RISCV::V8)
20083 .
Case(
"{v9}", RISCV::V9)
20084 .
Case(
"{v10}", RISCV::V10)
20085 .
Case(
"{v11}", RISCV::V11)
20086 .
Case(
"{v12}", RISCV::V12)
20087 .
Case(
"{v13}", RISCV::V13)
20088 .
Case(
"{v14}", RISCV::V14)
20089 .
Case(
"{v15}", RISCV::V15)
20090 .
Case(
"{v16}", RISCV::V16)
20091 .
Case(
"{v17}", RISCV::V17)
20092 .
Case(
"{v18}", RISCV::V18)
20093 .
Case(
"{v19}", RISCV::V19)
20094 .
Case(
"{v20}", RISCV::V20)
20095 .
Case(
"{v21}", RISCV::V21)
20096 .
Case(
"{v22}", RISCV::V22)
20097 .
Case(
"{v23}", RISCV::V23)
20098 .
Case(
"{v24}", RISCV::V24)
20099 .
Case(
"{v25}", RISCV::V25)
20100 .
Case(
"{v26}", RISCV::V26)
20101 .
Case(
"{v27}", RISCV::V27)
20102 .
Case(
"{v28}", RISCV::V28)
20103 .
Case(
"{v29}", RISCV::V29)
20104 .
Case(
"{v30}", RISCV::V30)
20105 .
Case(
"{v31}", RISCV::V31)
20107 if (VReg != RISCV::NoRegister) {
20108 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20109 return std::make_pair(VReg, &RISCV::VMRegClass);
20110 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20111 return std::make_pair(VReg, &RISCV::VRRegClass);
20112 for (
const auto *RC :
20113 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20114 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20115 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20116 return std::make_pair(VReg, RC);
20122 std::pair<Register, const TargetRegisterClass *> Res =
20128 if (Res.second == &RISCV::GPRF16RegClass ||
20129 Res.second == &RISCV::GPRF32RegClass ||
20130 Res.second == &RISCV::GPRPairRegClass)
20131 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20139 if (ConstraintCode.
size() == 1) {
20140 switch (ConstraintCode[0]) {
20155 if (Constraint.
size() == 1) {
20156 switch (Constraint[0]) {
20159 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20161 if (isInt<12>(CVal))
20174 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20176 if (isUInt<5>(CVal))
20194 if (Subtarget.hasStdExtZtso()) {
20210 if (Subtarget.hasStdExtZtso()) {
20218 if (Subtarget.enableSeqCstTrailingFence() && isa<StoreInst>(Inst) &&
20235 if (Subtarget.hasForcedAtomics())
20240 if (Subtarget.hasStdExtZacas() &&
20241 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20247 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20260 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20262 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20264 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20266 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20268 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20270 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20272 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20274 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20283 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20285 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20287 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20289 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20291 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20293 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20295 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20297 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20323 unsigned XLen = Subtarget.
getXLen();
20347 unsigned ValWidth =
20352 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20355 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20367 if (Subtarget.hasForcedAtomics())
20371 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20380 unsigned XLen = Subtarget.
getXLen();
20382 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20387 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20393 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20400 EVT DataVT)
const {
20416 return Subtarget.hasStdExtZfhmin();
20418 return Subtarget.hasStdExtF();
20420 return Subtarget.hasStdExtD();
20452 "RVVBitsPerBlock changed, audit needed");
20461 if (!Subtarget.hasVendorXTHeadMemIdx())
20467 Base =
Op->getOperand(0);
20469 int64_t RHSC =
RHS->getSExtValue();
20475 bool isLegalIndexedOffset =
false;
20476 for (
unsigned i = 0; i < 4; i++)
20477 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20478 isLegalIndexedOffset =
true;
20482 if (!isLegalIndexedOffset)
20499 VT = LD->getMemoryVT();
20500 Ptr = LD->getBasePtr();
20501 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20502 VT = ST->getMemoryVT();
20503 Ptr = ST->getBasePtr();
20522 VT = LD->getMemoryVT();
20523 Ptr = LD->getBasePtr();
20524 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20525 VT = ST->getMemoryVT();
20526 Ptr = ST->getBasePtr();
20569 const Constant *PersonalityFn)
const {
20574 const Constant *PersonalityFn)
const {
20598 const bool HasExtMOrZmmul =
20599 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul();
20608 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
20610 const APInt &Imm = ConstNode->getAPIntValue();
20611 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
20612 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
20616 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
20617 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
20618 (Imm - 8).isPowerOf2()))
20623 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
20624 ConstNode->hasOneUse()) {
20625 APInt ImmS = Imm.ashr(Imm.countr_zero());
20626 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
20627 (1 - ImmS).isPowerOf2())
20651 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
20660 unsigned *
Fast)
const {
20663 *
Fast = Subtarget.enableUnalignedScalarMem();
20664 return Subtarget.enableUnalignedScalarMem();
20680 *
Fast = Subtarget.enableUnalignedVectorMem();
20681 return Subtarget.enableUnalignedVectorMem();
20690 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
20702 if (
Op.size() < MinVLenInBytes)
20712 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
20716 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
20718 if (
Op.isFixedDstAlign())
20719 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
20721 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
20729 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
20730 bool IsABIRegCopy =
CC.has_value();
20732 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20733 PartVT == MVT::f32) {
20751 if (PartVTBitSize % ValueVTBitSize == 0) {
20752 assert(PartVTBitSize >= ValueVTBitSize);
20759 if (ValueEltVT != PartEltVT) {
20760 if (PartVTBitSize > ValueVTBitSize) {
20762 assert(Count != 0 &&
"The number of element should not be zero.");
20763 EVT SameEltTypeVT =
20784 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
20785 bool IsABIRegCopy =
CC.has_value();
20786 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20787 PartVT == MVT::f32) {
20804 if (PartVTBitSize % ValueVTBitSize == 0) {
20805 assert(PartVTBitSize >= ValueVTBitSize);
20806 EVT SameEltTypeVT = ValueVT;
20813 if (ValueEltVT != PartEltVT) {
20815 assert(Count != 0 &&
"The number of element should not be zero.");
20832 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
20839 unsigned Opc =
N->getOpcode();
20863 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
20877 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
20882 if (FVTy->getNumElements() < 2)
20892 return Factor * LMUL <= 8;
20896 Align Alignment)
const {
20908 if (!Subtarget.enableUnalignedVectorMem() &&
20916 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
20917 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
20918 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
20919 Intrinsic::riscv_seg8_load};
20938 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
20948 {VTy, LI->getPointerOperandType(), XLenTy});
20950 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
20955 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
20957 Shuffles[i]->replaceAllUsesWith(SubVec);
20964 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
20965 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
20966 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
20967 Intrinsic::riscv_seg8_store};
20987 unsigned Factor)
const {
20989 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
20992 ShuffleVTy->getNumElements() / Factor);
20994 SI->getPointerAddressSpace(),
20995 SI->getModule()->getDataLayout()))
21002 {VTy, SI->getPointerOperandType(), XLenTy});
21007 for (
unsigned i = 0; i < Factor; i++) {
21016 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21017 Ops.
append({SI->getPointerOperand(), VL});
21033 unsigned Factor = 2;
21048 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21051 {ResVTy, LI->getPointerOperandType(), XLenTy});
21052 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21055 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21056 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21057 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21058 Intrinsic::riscv_vlseg8};
21083 unsigned Factor = 2;
21089 SI->getPointerAddressSpace(),
21090 SI->getModule()->getDataLayout()))
21097 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21100 {InVTy, SI->getPointerOperandType(), XLenTy});
21101 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21104 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21105 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21106 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21107 Intrinsic::riscv_vsseg8};
21115 SI->getPointerOperand(), VL});
21125 "Invalid call instruction for a KCFI check");
21127 MBBI->getOpcode()));
21130 Target.setIsRenamable(
false);
21138#define GET_REGISTER_MATCHER
21139#include "RISCVGenAsmMatcher.inc"
21145 if (Reg == RISCV::NoRegister)
21147 if (Reg == RISCV::NoRegister)
21159 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21161 if (NontemporalInfo ==
nullptr)
21169 int NontemporalLevel = 5;
21170 const MDNode *RISCVNontemporalInfo =
21171 I.getMetadata(
"riscv-nontemporal-domain");
21172 if (RISCVNontemporalInfo !=
nullptr)
21175 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21179 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21180 "RISC-V target doesn't support this non-temporal domain.");
21182 NontemporalLevel -= 2;
21184 if (NontemporalLevel & 0b1)
21186 if (NontemporalLevel & 0b10)
21199 return TargetFlags;
21209 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21212 return Subtarget.hasStdExtZbb() &&
21225 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21226 Op == Instruction::And ||
Op == Instruction::Or ||
21227 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21228 Op == Instruction::ShuffleVector ||
Op == Instruction::Load)
21236 !isa<ReturnInst>(&Inst))
21239 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21240 if (AI->getAllocatedType()->isScalableTy())
21248RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21256 if (!Subtarget.hasShortForwardBranchOpt())
21258 EVT VT =
N->getValueType(0);
21259 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21263 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21268bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21269 EVT VT,
const APInt &AndMask)
const {
21270 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21271 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21275unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21280template <
typename Arg>
21281void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21284 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21286 auto It = ArgList.
begin();
21287 MVT FirstArgRegType = It->VT;
21290 if (It == ArgList.
end() || It->Flags.isSplit())
21302 for (; It != ArgList.
end(); ++It)
21303 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21309 if (isHomogeneousScalableVectorType(ArgList)) {
21311 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21314 bool FirstVMaskAssigned =
false;
21315 for (
const auto &OutArg : ArgList) {
21316 MVT RegisterVT = OutArg.VT;
21326 RVVArgInfos.push_back({1, RegisterVT,
true});
21327 FirstVMaskAssigned =
true;
21331 RVVArgInfos.push_back({1, RegisterVT,
false});
21338void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21343 bool FirstVMaskAssigned =
false;
21344 for (
Type *Ty : TypeList) {
21354 RVVArgInfos.push_back(
21360 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21375 if (!FirstVMaskAssigned &&
21377 RVVArgInfos.push_back({1, RegisterVT,
true});
21378 FirstVMaskAssigned =
true;
21382 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21388void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21389 unsigned StartReg) {
21390 assert((StartReg % LMul) == 0 &&
21391 "Start register number should be multiple of lmul");
21410 for (
unsigned i = 0; i < NF; ++i)
21412 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
21414 AllocatedPhysRegs.push_back(
MCPhysReg());
21420void RVVArgDispatcher::compute() {
21422 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
21425 AllocatedPhysRegs.push_back(RISCV::V0);
21431 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
21432 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
21433 StartReg += RegsNeeded) {
21434 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
21435 if ((AssignedMap & Map) == 0) {
21436 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
21437 AssignedMap |=
Map;
21442 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
21445 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
21446 allocate(RVVArgInfos[i]);
21450 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
21451 return AllocatedPhysRegs[CurIdx++];
21456#define GET_RISCVVIntrinsicsTable_IMPL
21457#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static bool IsSelect(MachineInstr &MI)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Helper struct to store a base, index and offset that forms an address.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
bool hasScalarOperand() const
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)