38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
274 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
286 if (!Subtarget.hasStdExtZbb())
290 if (Subtarget.hasStdExtZbb()) {
298 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
303 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
307 }
else if (Subtarget.
is64Bit()) {
317 if (!Subtarget.hasStdExtM()) {
323 }
else if (Subtarget.
is64Bit()) {
326 {MVT::i8, MVT::i16, MVT::i32},
Custom);
343 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
346 }
else if (Subtarget.hasVendorXTHeadBb()) {
350 }
else if (Subtarget.hasVendorXCVbitmanip()) {
361 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
362 Subtarget.hasVendorXTHeadBb())
367 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
368 Subtarget.hasVendorXTHeadBb())
373 if (Subtarget.hasVendorXCVbitmanip()) {
381 if (Subtarget.hasStdExtZbb()) {
394 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
400 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
401 Subtarget.hasVendorXCVbitmanip()) {
408 if (!Subtarget.hasStdExtZbb())
420 !Subtarget.hasShortForwardBranchOpt())
424 if (Subtarget.hasShortForwardBranchOpt())
427 if (!Subtarget.hasVendorXTHeadCondMov()) {
433 static const unsigned FPLegalNodeTypes[] = {
446 static const unsigned FPOpToExpand[] = {
450 static const unsigned FPRndMode[] = {
457 static const unsigned ZfhminZfbfminPromoteOps[] = {
468 if (Subtarget.hasStdExtZfbfmin()) {
527 if (!Subtarget.hasStdExtZfa())
551 if (Subtarget.hasStdExtZfa())
566 if (Subtarget.hasStdExtZfa()) {
642 if (Subtarget.hasStdExtZicbop()) {
646 if (Subtarget.hasStdExtA()) {
648 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
652 }
else if (Subtarget.hasForcedAtomics()) {
673 {MVT::i8, MVT::i16},
Custom);
684 static const unsigned IntegerVPOps[] = {
685 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
686 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
687 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
688 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
689 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
690 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
691 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
692 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
693 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
694 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
695 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
696 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
697 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
700 static const unsigned FloatingPointVPOps[] = {
701 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
702 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
703 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
704 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
705 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
706 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
707 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
708 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
709 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
710 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
711 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
712 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
713 ISD::EXPERIMENTAL_VP_SPLICE};
715 static const unsigned IntegerVecReduceOps[] = {
720 static const unsigned FloatingPointVecReduceOps[] = {
733 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
734 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
735 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
739 for (
MVT VT : BoolVecVTs) {
766 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
790 ISD::VP_TRUNCATE, ISD::VP_SETCC},
806 for (
MVT VT : IntVecVTs) {
817 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
864 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
865 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
889 if (Subtarget.hasStdExtZvkb()) {
897 if (Subtarget.hasStdExtZvbb()) {
901 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
907 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
916 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
936 static const unsigned ZvfhminPromoteOps[] = {
946 static const unsigned ZvfhminPromoteVPOps[] = {
947 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
948 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
949 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
950 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
951 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
952 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
953 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
954 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
958 const auto SetCommonVFPActions = [&](
MVT VT) {
1003 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1004 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1035 const auto SetCommonVFPExtLoadTruncStoreActions =
1037 for (
auto SmallVT : SmallerVTs) {
1044 for (
MVT VT : F16VecVTs) {
1047 SetCommonVFPActions(VT);
1050 for (
MVT VT : F16VecVTs) {
1061 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1072 if (VT == MVT::nxv32f16) {
1085 for (
MVT VT : F32VecVTs) {
1088 SetCommonVFPActions(VT);
1089 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1094 for (
MVT VT : F64VecVTs) {
1097 SetCommonVFPActions(VT);
1098 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1099 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1105 if (!useRVVForFixedLengthVectorVT(VT))
1148 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1175 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1197 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1198 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1235 if (Subtarget.hasStdExtZvkb())
1238 if (Subtarget.hasStdExtZvbb()) {
1260 if (!useRVVForFixedLengthVectorVT(VT))
1281 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1313 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1314 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1365 if (Subtarget.hasStdExtA()) {
1371 if (Subtarget.hasForcedAtomics()) {
1381 if (Subtarget.hasVendorXTHeadMemIdx()) {
1413 if (Subtarget.hasStdExtZbb())
1416 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1419 if (Subtarget.hasStdExtZbkb())
1431 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1434 if (Subtarget.hasVendorXTHeadMemPair())
1457MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1462bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1464 bool IsScalable)
const {
1471 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1488bool RISCVTargetLowering::shouldExpandCttzElements(
EVT VT)
const {
1496 unsigned Intrinsic)
const {
1497 auto &
DL =
I.getModule()->getDataLayout();
1499 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1500 bool IsUnitStrided,
bool UsePtrVal =
false) {
1505 Info.ptrVal =
I.getArgOperand(PtrOp);
1507 Info.fallbackAddressSpace =
1508 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1512 MemTy =
I.getArgOperand(0)->getType();
1515 MemTy =
I.getType();
1530 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1534 switch (Intrinsic) {
1537 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1538 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1539 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1540 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1541 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1542 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1543 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1544 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1545 case Intrinsic::riscv_masked_cmpxchg_i32:
1547 Info.memVT = MVT::i32;
1548 Info.ptrVal =
I.getArgOperand(0);
1554 case Intrinsic::riscv_masked_strided_load:
1555 return SetRVVLoadStoreInfo( 1,
false,
1557 case Intrinsic::riscv_masked_strided_store:
1558 return SetRVVLoadStoreInfo( 1,
true,
1560 case Intrinsic::riscv_seg2_load:
1561 case Intrinsic::riscv_seg3_load:
1562 case Intrinsic::riscv_seg4_load:
1563 case Intrinsic::riscv_seg5_load:
1564 case Intrinsic::riscv_seg6_load:
1565 case Intrinsic::riscv_seg7_load:
1566 case Intrinsic::riscv_seg8_load:
1567 return SetRVVLoadStoreInfo( 0,
false,
1569 case Intrinsic::riscv_seg2_store:
1570 case Intrinsic::riscv_seg3_store:
1571 case Intrinsic::riscv_seg4_store:
1572 case Intrinsic::riscv_seg5_store:
1573 case Intrinsic::riscv_seg6_store:
1574 case Intrinsic::riscv_seg7_store:
1575 case Intrinsic::riscv_seg8_store:
1577 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1580 case Intrinsic::riscv_vle:
1581 case Intrinsic::riscv_vle_mask:
1582 case Intrinsic::riscv_vleff:
1583 case Intrinsic::riscv_vleff_mask:
1584 return SetRVVLoadStoreInfo( 1,
1588 case Intrinsic::riscv_vse:
1589 case Intrinsic::riscv_vse_mask:
1590 return SetRVVLoadStoreInfo( 1,
1594 case Intrinsic::riscv_vlse:
1595 case Intrinsic::riscv_vlse_mask:
1596 case Intrinsic::riscv_vloxei:
1597 case Intrinsic::riscv_vloxei_mask:
1598 case Intrinsic::riscv_vluxei:
1599 case Intrinsic::riscv_vluxei_mask:
1600 return SetRVVLoadStoreInfo( 1,
1603 case Intrinsic::riscv_vsse:
1604 case Intrinsic::riscv_vsse_mask:
1605 case Intrinsic::riscv_vsoxei:
1606 case Intrinsic::riscv_vsoxei_mask:
1607 case Intrinsic::riscv_vsuxei:
1608 case Intrinsic::riscv_vsuxei_mask:
1609 return SetRVVLoadStoreInfo( 1,
1612 case Intrinsic::riscv_vlseg2:
1613 case Intrinsic::riscv_vlseg3:
1614 case Intrinsic::riscv_vlseg4:
1615 case Intrinsic::riscv_vlseg5:
1616 case Intrinsic::riscv_vlseg6:
1617 case Intrinsic::riscv_vlseg7:
1618 case Intrinsic::riscv_vlseg8:
1619 case Intrinsic::riscv_vlseg2ff:
1620 case Intrinsic::riscv_vlseg3ff:
1621 case Intrinsic::riscv_vlseg4ff:
1622 case Intrinsic::riscv_vlseg5ff:
1623 case Intrinsic::riscv_vlseg6ff:
1624 case Intrinsic::riscv_vlseg7ff:
1625 case Intrinsic::riscv_vlseg8ff:
1626 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1629 case Intrinsic::riscv_vlseg2_mask:
1630 case Intrinsic::riscv_vlseg3_mask:
1631 case Intrinsic::riscv_vlseg4_mask:
1632 case Intrinsic::riscv_vlseg5_mask:
1633 case Intrinsic::riscv_vlseg6_mask:
1634 case Intrinsic::riscv_vlseg7_mask:
1635 case Intrinsic::riscv_vlseg8_mask:
1636 case Intrinsic::riscv_vlseg2ff_mask:
1637 case Intrinsic::riscv_vlseg3ff_mask:
1638 case Intrinsic::riscv_vlseg4ff_mask:
1639 case Intrinsic::riscv_vlseg5ff_mask:
1640 case Intrinsic::riscv_vlseg6ff_mask:
1641 case Intrinsic::riscv_vlseg7ff_mask:
1642 case Intrinsic::riscv_vlseg8ff_mask:
1643 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1646 case Intrinsic::riscv_vlsseg2:
1647 case Intrinsic::riscv_vlsseg3:
1648 case Intrinsic::riscv_vlsseg4:
1649 case Intrinsic::riscv_vlsseg5:
1650 case Intrinsic::riscv_vlsseg6:
1651 case Intrinsic::riscv_vlsseg7:
1652 case Intrinsic::riscv_vlsseg8:
1653 case Intrinsic::riscv_vloxseg2:
1654 case Intrinsic::riscv_vloxseg3:
1655 case Intrinsic::riscv_vloxseg4:
1656 case Intrinsic::riscv_vloxseg5:
1657 case Intrinsic::riscv_vloxseg6:
1658 case Intrinsic::riscv_vloxseg7:
1659 case Intrinsic::riscv_vloxseg8:
1660 case Intrinsic::riscv_vluxseg2:
1661 case Intrinsic::riscv_vluxseg3:
1662 case Intrinsic::riscv_vluxseg4:
1663 case Intrinsic::riscv_vluxseg5:
1664 case Intrinsic::riscv_vluxseg6:
1665 case Intrinsic::riscv_vluxseg7:
1666 case Intrinsic::riscv_vluxseg8:
1667 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1670 case Intrinsic::riscv_vlsseg2_mask:
1671 case Intrinsic::riscv_vlsseg3_mask:
1672 case Intrinsic::riscv_vlsseg4_mask:
1673 case Intrinsic::riscv_vlsseg5_mask:
1674 case Intrinsic::riscv_vlsseg6_mask:
1675 case Intrinsic::riscv_vlsseg7_mask:
1676 case Intrinsic::riscv_vlsseg8_mask:
1677 case Intrinsic::riscv_vloxseg2_mask:
1678 case Intrinsic::riscv_vloxseg3_mask:
1679 case Intrinsic::riscv_vloxseg4_mask:
1680 case Intrinsic::riscv_vloxseg5_mask:
1681 case Intrinsic::riscv_vloxseg6_mask:
1682 case Intrinsic::riscv_vloxseg7_mask:
1683 case Intrinsic::riscv_vloxseg8_mask:
1684 case Intrinsic::riscv_vluxseg2_mask:
1685 case Intrinsic::riscv_vluxseg3_mask:
1686 case Intrinsic::riscv_vluxseg4_mask:
1687 case Intrinsic::riscv_vluxseg5_mask:
1688 case Intrinsic::riscv_vluxseg6_mask:
1689 case Intrinsic::riscv_vluxseg7_mask:
1690 case Intrinsic::riscv_vluxseg8_mask:
1691 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1694 case Intrinsic::riscv_vsseg2:
1695 case Intrinsic::riscv_vsseg3:
1696 case Intrinsic::riscv_vsseg4:
1697 case Intrinsic::riscv_vsseg5:
1698 case Intrinsic::riscv_vsseg6:
1699 case Intrinsic::riscv_vsseg7:
1700 case Intrinsic::riscv_vsseg8:
1701 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1704 case Intrinsic::riscv_vsseg2_mask:
1705 case Intrinsic::riscv_vsseg3_mask:
1706 case Intrinsic::riscv_vsseg4_mask:
1707 case Intrinsic::riscv_vsseg5_mask:
1708 case Intrinsic::riscv_vsseg6_mask:
1709 case Intrinsic::riscv_vsseg7_mask:
1710 case Intrinsic::riscv_vsseg8_mask:
1711 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1714 case Intrinsic::riscv_vssseg2:
1715 case Intrinsic::riscv_vssseg3:
1716 case Intrinsic::riscv_vssseg4:
1717 case Intrinsic::riscv_vssseg5:
1718 case Intrinsic::riscv_vssseg6:
1719 case Intrinsic::riscv_vssseg7:
1720 case Intrinsic::riscv_vssseg8:
1721 case Intrinsic::riscv_vsoxseg2:
1722 case Intrinsic::riscv_vsoxseg3:
1723 case Intrinsic::riscv_vsoxseg4:
1724 case Intrinsic::riscv_vsoxseg5:
1725 case Intrinsic::riscv_vsoxseg6:
1726 case Intrinsic::riscv_vsoxseg7:
1727 case Intrinsic::riscv_vsoxseg8:
1728 case Intrinsic::riscv_vsuxseg2:
1729 case Intrinsic::riscv_vsuxseg3:
1730 case Intrinsic::riscv_vsuxseg4:
1731 case Intrinsic::riscv_vsuxseg5:
1732 case Intrinsic::riscv_vsuxseg6:
1733 case Intrinsic::riscv_vsuxseg7:
1734 case Intrinsic::riscv_vsuxseg8:
1735 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1738 case Intrinsic::riscv_vssseg2_mask:
1739 case Intrinsic::riscv_vssseg3_mask:
1740 case Intrinsic::riscv_vssseg4_mask:
1741 case Intrinsic::riscv_vssseg5_mask:
1742 case Intrinsic::riscv_vssseg6_mask:
1743 case Intrinsic::riscv_vssseg7_mask:
1744 case Intrinsic::riscv_vssseg8_mask:
1745 case Intrinsic::riscv_vsoxseg2_mask:
1746 case Intrinsic::riscv_vsoxseg3_mask:
1747 case Intrinsic::riscv_vsoxseg4_mask:
1748 case Intrinsic::riscv_vsoxseg5_mask:
1749 case Intrinsic::riscv_vsoxseg6_mask:
1750 case Intrinsic::riscv_vsoxseg7_mask:
1751 case Intrinsic::riscv_vsoxseg8_mask:
1752 case Intrinsic::riscv_vsuxseg2_mask:
1753 case Intrinsic::riscv_vsuxseg3_mask:
1754 case Intrinsic::riscv_vsuxseg4_mask:
1755 case Intrinsic::riscv_vsuxseg5_mask:
1756 case Intrinsic::riscv_vsuxseg6_mask:
1757 case Intrinsic::riscv_vsuxseg7_mask:
1758 case Intrinsic::riscv_vsuxseg8_mask:
1759 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1796 return isInt<12>(Imm);
1800 return isInt<12>(Imm);
1813 return (SrcBits == 64 && DestBits == 32);
1824 return (SrcBits == 64 && DestBits == 32);
1831 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1832 EVT MemVT = LD->getMemoryVT();
1833 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1843 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1851 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1855 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1856 Subtarget.hasVendorXCVbitmanip();
1867 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1872 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1876 EVT VT =
Y.getValueType();
1882 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1883 !isa<ConstantSDNode>(
Y);
1888 if (Subtarget.hasStdExtZbs())
1889 return X.getValueType().isScalarInteger();
1890 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1892 if (Subtarget.hasVendorXTHeadBs())
1893 return C !=
nullptr;
1895 return C &&
C->getAPIntValue().ule(10);
1915 if (BitSize > Subtarget.
getXLen())
1919 int64_t Val = Imm.getSExtValue();
1927 if (!Subtarget.enableUnalignedScalarMem())
1943 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1950 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1954 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1966 case Instruction::Add:
1967 case Instruction::Sub:
1968 case Instruction::Mul:
1969 case Instruction::And:
1970 case Instruction::Or:
1971 case Instruction::Xor:
1972 case Instruction::FAdd:
1973 case Instruction::FSub:
1974 case Instruction::FMul:
1975 case Instruction::FDiv:
1976 case Instruction::ICmp:
1977 case Instruction::FCmp:
1979 case Instruction::Shl:
1980 case Instruction::LShr:
1981 case Instruction::AShr:
1982 case Instruction::UDiv:
1983 case Instruction::SDiv:
1984 case Instruction::URem:
1985 case Instruction::SRem:
1986 return Operand == 1;
2000 auto *II = dyn_cast<IntrinsicInst>(
I);
2004 switch (II->getIntrinsicID()) {
2005 case Intrinsic::fma:
2006 case Intrinsic::vp_fma:
2007 return Operand == 0 || Operand == 1;
2008 case Intrinsic::vp_shl:
2009 case Intrinsic::vp_lshr:
2010 case Intrinsic::vp_ashr:
2011 case Intrinsic::vp_udiv:
2012 case Intrinsic::vp_sdiv:
2013 case Intrinsic::vp_urem:
2014 case Intrinsic::vp_srem:
2015 case Intrinsic::ssub_sat:
2016 case Intrinsic::vp_ssub_sat:
2017 case Intrinsic::usub_sat:
2018 case Intrinsic::vp_usub_sat:
2019 return Operand == 1;
2021 case Intrinsic::vp_add:
2022 case Intrinsic::vp_mul:
2023 case Intrinsic::vp_and:
2024 case Intrinsic::vp_or:
2025 case Intrinsic::vp_xor:
2026 case Intrinsic::vp_fadd:
2027 case Intrinsic::vp_fmul:
2028 case Intrinsic::vp_icmp:
2029 case Intrinsic::vp_fcmp:
2030 case Intrinsic::smin:
2031 case Intrinsic::vp_smin:
2032 case Intrinsic::umin:
2033 case Intrinsic::vp_umin:
2034 case Intrinsic::smax:
2035 case Intrinsic::vp_smax:
2036 case Intrinsic::umax:
2037 case Intrinsic::vp_umax:
2038 case Intrinsic::sadd_sat:
2039 case Intrinsic::vp_sadd_sat:
2040 case Intrinsic::uadd_sat:
2041 case Intrinsic::vp_uadd_sat:
2043 case Intrinsic::vp_sub:
2044 case Intrinsic::vp_fsub:
2045 case Intrinsic::vp_fdiv:
2046 return Operand == 0 || Operand == 1;
2067 if (!Subtarget.sinkSplatOperands())
2070 for (
auto OpIdx :
enumerate(
I->operands())) {
2074 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2076 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2085 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2090 for (
Use &U :
Op->uses()) {
2142 if (!Subtarget.hasStdExtZfa())
2143 return std::make_pair(-1,
false);
2145 bool IsSupportedVT =
false;
2146 if (VT == MVT::f16) {
2147 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2148 }
else if (VT == MVT::f32) {
2149 IsSupportedVT =
true;
2150 }
else if (VT == MVT::f64) {
2151 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2152 IsSupportedVT =
true;
2156 return std::make_pair(-1,
false);
2159 if (
Index < 0 && Imm.isNegative())
2163 return std::make_pair(
Index,
false);
2167 bool ForCodeSize)
const {
2168 bool IsLegalVT =
false;
2171 else if (VT == MVT::f32)
2173 else if (VT == MVT::f64)
2175 else if (VT == MVT::bf16)
2176 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2188 return Imm.isZero();
2192 if (Imm.isNegZero())
2205 unsigned Index)
const {
2218 if (EltVT == MVT::i1)
2231 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2238 if ((ResElts * 2) != SrcElts)
2282 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2284 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2287 IntermediateVT = MVT::i64;
2290 RegisterVT = MVT::i64;
2305 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2311 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2324 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2325 int64_t
C = RHSC->getSExtValue();
2367 switch (KnownSize) {
2395 return RISCV::VRRegClassID;
2397 return RISCV::VRM2RegClassID;
2399 return RISCV::VRM4RegClassID;
2401 return RISCV::VRM8RegClassID;
2411 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2412 "Unexpected subreg numbering");
2413 return RISCV::sub_vrm1_0 +
Index;
2416 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2417 "Unexpected subreg numbering");
2418 return RISCV::sub_vrm2_0 +
Index;
2421 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2422 "Unexpected subreg numbering");
2423 return RISCV::sub_vrm4_0 +
Index;
2430 return RISCV::VRRegClassID;
2439std::pair<unsigned, unsigned>
2441 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2443 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2444 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2445 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2446 "Register classes not ordered");
2455 unsigned SubRegIdx = RISCV::NoSubRegister;
2456 for (
const unsigned RCID :
2457 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2458 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2462 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2467 return {SubRegIdx, InsertExtractIdx};
2472bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2501unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2508 "Unexpected opcode");
2510 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2512 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2515 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2585bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2586 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2595 "Expected legal fixed length vector!");
2598 unsigned MaxELen = Subtarget.
getELen();
2631 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2638 "Expected to convert into a scalable vector!");
2639 assert(V.getValueType().isFixedLengthVector() &&
2640 "Expected a fixed length vector operand!");
2650 "Expected to convert into a fixed length vector!");
2651 assert(V.getValueType().isScalableVector() &&
2652 "Expected a scalable vector operand!");
2680 const auto [MinVLMAX, MaxVLMAX] =
2682 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2688static std::pair<SDValue, SDValue>
2697static std::pair<SDValue, SDValue>
2710static std::pair<SDValue, SDValue>
2727std::pair<unsigned, unsigned>
2743 return std::make_pair(MinVLMAX, MaxVLMAX);
2755 EVT VT,
unsigned DefinedValues)
const {
2769 std::tie(LMul, Fractional) =
2772 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2774 Cost = (LMul * DLenFactor);
2819 MVT DstVT =
Op.getSimpleValueType();
2820 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2828 Src.getValueType() == MVT::bf16) {
2835 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2843 Opc,
DL, DstVT, Src,
2857 MVT SrcVT = Src.getSimpleValueType();
2863 if (SatVT != DstEltVT)
2867 if (SrcEltSize > (2 * DstEltSize))
2870 MVT DstContainerVT = DstVT;
2871 MVT SrcContainerVT = SrcVT;
2877 "Expected same element count");
2886 {Src, Src, DAG.getCondCode(ISD::SETNE),
2887 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2891 if (DstEltSize > (2 * SrcEltSize)) {
2905 Res, DAG.
getUNDEF(DstContainerVT), VL);
2917 case ISD::VP_FROUNDEVEN:
2921 case ISD::VP_FROUNDTOZERO:
2925 case ISD::VP_FFLOOR:
2933 case ISD::VP_FROUND:
2949 MVT VT =
Op.getSimpleValueType();
2956 MVT ContainerVT = VT;
2963 if (
Op->isVPOpcode()) {
2964 Mask =
Op.getOperand(1);
2968 VL =
Op.getOperand(2);
2990 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3004 switch (
Op.getOpcode()) {
3010 case ISD::VP_FFLOOR:
3013 case ISD::VP_FROUND:
3014 case ISD::VP_FROUNDEVEN:
3015 case ISD::VP_FROUNDTOZERO: {
3031 case ISD::VP_FNEARBYINT:
3044 Src, Src, Mask, VL);
3059 MVT VT =
Op.getSimpleValueType();
3063 MVT ContainerVT = VT;
3075 MVT MaskVT = Mask.getSimpleValueType();
3078 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3079 DAG.getUNDEF(MaskVT), Mask, VL});
3083 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
3084 Chain = Src.getValue(1);
3100 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3112 switch (
Op.getOpcode()) {
3123 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3129 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3133 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3142 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3143 Truncated, Mask, VL);
3149 Src, Src, Mask, VL);
3159 MVT VT =
Op.getSimpleValueType();
3187 MVT VT =
Op.getSimpleValueType();
3192 MVT ContainerVT = VT;
3214 if (
Merge.isUndef())
3226 if (
Merge.isUndef())
3235 "Unexpected vector MVT");
3263 return std::nullopt;
3278 unsigned EltSizeInBits) {
3281 return std::nullopt;
3282 bool IsInteger =
Op.getValueType().isInteger();
3284 std::optional<unsigned> SeqStepDenom;
3285 std::optional<int64_t> SeqStepNum, SeqAddend;
3286 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3287 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3292 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3294 if (Elt.isUndef()) {
3295 Elts[
Idx] = std::nullopt;
3299 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3304 return std::nullopt;
3305 Elts[
Idx] = *ExactInteger;
3318 unsigned IdxDiff =
Idx - PrevElt->second;
3319 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3327 int64_t Remainder = ValDiff % IdxDiff;
3329 if (Remainder != ValDiff) {
3332 return std::nullopt;
3338 SeqStepNum = ValDiff;
3339 else if (ValDiff != SeqStepNum)
3340 return std::nullopt;
3343 SeqStepDenom = IdxDiff;
3344 else if (IdxDiff != *SeqStepDenom)
3345 return std::nullopt;
3349 if (!PrevElt || PrevElt->first != *Elt)
3350 PrevElt = std::make_pair(*Elt,
Idx);
3354 if (!SeqStepNum || !SeqStepDenom)
3355 return std::nullopt;
3363 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3364 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3367 else if (Addend != SeqAddend)
3368 return std::nullopt;
3371 assert(SeqAddend &&
"Must have an addend if we have a step");
3373 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3394 MVT ContainerVT = VT;
3422 MVT VT =
Op.getSimpleValueType();
3434 unsigned MostCommonCount = 0;
3436 unsigned NumUndefElts =
3444 unsigned NumScalarLoads = 0;
3450 ValueCounts.
insert(std::make_pair(V, 0));
3451 unsigned &Count = ValueCounts[V];
3453 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3454 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3459 if (++Count >= MostCommonCount) {
3461 MostCommonCount = Count;
3465 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3466 unsigned NumDefElts = NumElts - NumUndefElts;
3467 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3473 ((MostCommonCount > DominantValueCountThreshold) ||
3486 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3487 LastOp != DominantValue) {
3496 Processed.insert(LastOp);
3501 const SDValue &V = OpIdx.value();
3502 if (V.isUndef() || !Processed.insert(V).second)
3504 if (ValueCounts[V] == 1) {
3513 return DAG.getConstant(V == V1, DL, XLenVT);
3529 MVT VT =
Op.getSimpleValueType();
3559 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3560 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3568 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3569 MVT IntegerViaVecVT =
3574 unsigned BitPos = 0, IntegerEltIdx = 0;
3577 for (
unsigned I = 0;
I < NumElts;) {
3579 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3580 Bits |= ((
uint64_t)BitValue << BitPos);
3586 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3587 if (NumViaIntegerBits <= 32)
3588 Bits = SignExtend64<32>(Bits);
3590 Elts[IntegerEltIdx] = Elt;
3599 if (NumElts < NumViaIntegerBits) {
3603 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3631 int64_t StepNumerator = SimpleVID->StepNumerator;
3632 unsigned StepDenominator = SimpleVID->StepDenominator;
3633 int64_t Addend = SimpleVID->Addend;
3635 assert(StepNumerator != 0 &&
"Invalid step");
3636 bool Negate =
false;
3637 int64_t SplatStepVal = StepNumerator;
3641 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3643 Negate = StepNumerator < 0;
3645 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3652 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3653 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3655 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3658 MVT VIDContainerVT =
3666 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3667 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3669 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3671 if (StepDenominator != 1) {
3676 if (Addend != 0 || Negate) {
3695 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3696 "Unexpected sequence type");
3700 unsigned ViaVecLen =
3704 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3707 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3708 const auto &SeqV = OpIdx.value();
3709 if (!SeqV.isUndef())
3711 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3716 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3717 SplatValue = SignExtend64<32>(SplatValue);
3739 const auto *BV = cast<BuildVectorSDNode>(
Op);
3742 BV->getRepeatedSequence(Sequence) &&
3743 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3744 unsigned SeqLen = Sequence.size();
3746 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3747 ViaIntVT == MVT::i64) &&
3748 "Unexpected sequence type");
3753 const unsigned RequiredVL = NumElts / SeqLen;
3754 const unsigned ViaVecLen =
3756 NumElts : RequiredVL;
3759 unsigned EltIdx = 0;
3760 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3764 for (
const auto &SeqV : Sequence) {
3765 if (!SeqV.isUndef())
3767 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3773 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3774 SplatValue = SignExtend64<32>(SplatValue);
3780 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3781 "Unexpected bitcast sequence");
3782 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3785 MVT ViaContainerVT =
3792 if (ViaVecLen != RequiredVL)
3809 if (EltBitSize - SignBits < 8) {
3813 Source, DAG, Subtarget);
3830 MVT VT =
Op.getSimpleValueType();
3902 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3906 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3922 unsigned NumUndefElts =
3924 unsigned NumDefElts = NumElts - NumUndefElts;
3925 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
3932 for (
unsigned i = 0; i < NumElts; i++) {
3934 if (i < NumElts / 2) {
3941 bool SelectMaskVal = (i < NumElts / 2);
3944 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
3945 MaskVals.
size() == NumElts);
3980 unsigned UndefCount = 0;
3987 LinearBudget -= PerSlideCost;
3990 LinearBudget -= PerSlideCost;
3993 LinearBudget -= PerSlideCost;
3996 if (LinearBudget < 0)
4001 "Illegal type which will result in reserved encoding");
4026 Vec,
Offset, Mask, VL, Policy);
4039 Vec,
Offset, Mask, VL, Policy);
4049 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4050 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4051 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4054 if ((LoC >> 31) == HiC)
4065 (isa<RegisterSDNode>(VL) &&
4066 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4068 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4083 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4084 Hi.getConstantOperandVal(1) == 31)
4103 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4115 bool HasPassthru = Passthru && !Passthru.
isUndef();
4116 if (!HasPassthru && !Passthru)
4124 if (Scalar.getValueType().bitsLE(XLenVT)) {
4131 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4135 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4136 "Unexpected scalar for splat lowering!");
4160 SDValue ExtractedVal = Scalar.getOperand(0);
4165 MVT ExtractedContainerVT = ExtractedVT;
4168 DAG, ExtractedContainerVT, Subtarget);
4170 ExtractedVal, DAG, Subtarget);
4172 if (ExtractedContainerVT.
bitsLE(VT))
4187 if (!Scalar.getValueType().bitsLE(XLenVT))
4190 VT,
DL, DAG, Subtarget);
4198 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4224 if (Src != V2.getOperand(0))
4228 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4233 V2.getConstantOperandVal(1) != Mask.size())
4237 if (Mask[0] != 0 && Mask[0] != 1)
4242 for (
unsigned i = 1; i != Mask.size(); ++i)
4243 if (Mask[i] != Mask[i - 1] + 2)
4261 int Size = Mask.size();
4263 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4269 EvenSrc = StartIndexes[0];
4270 OddSrc = StartIndexes[1];
4273 if (EvenSrc != 0 && OddSrc != 0)
4283 int HalfNumElts = NumElts / 2;
4284 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4300 int Size = Mask.size();
4312 for (
int i = 0; i !=
Size; ++i) {
4318 int StartIdx = i - (M %
Size);
4326 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4329 Rotation = CandidateRotation;
4330 else if (Rotation != CandidateRotation)
4335 int MaskSrc = M <
Size ? 0 : 1;
4340 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4345 TargetSrc = MaskSrc;
4346 else if (TargetSrc != MaskSrc)
4353 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4354 assert((LoSrc >= 0 || HiSrc >= 0) &&
4355 "Failed to find a rotated input vector!");
4370 MVT ContainerVT = VT;
4373 assert(Src.getSimpleValueType().isFixedLengthVector());
4377 MVT SrcContainerVT =
4390 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4397 unsigned Shift = EvenElts ? 0 : EltBits;
4403 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4429 auto findNonEXTRACT_SUBVECTORParent =
4430 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4435 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4436 Offset += Parent.getConstantOperandVal(1);
4437 Parent = Parent.getOperand(0);
4439 return std::make_pair(Parent,
Offset);
4442 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4443 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4452 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4453 if (NewMask[i] == -1)
4456 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4457 NewMask[i] = NewMask[i] + V1IndexOffset;
4461 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4467 if (NewMask[0] <= 0)
4471 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4472 if (NewMask[i - 1] + 1 != NewMask[i])
4476 MVT SrcVT = Src.getSimpleValueType();
4507 int NumSubElts,
Index;
4512 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4513 SDValue InPlace = OpsSwapped ? V2 : V1;
4514 SDValue ToInsert = OpsSwapped ? V1 : V2;
4524 if (NumSubElts +
Index >= (
int)NumElts)
4538 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4550 bool OpsSwapped =
false;
4551 if (!isa<BuildVectorSDNode>(V1)) {
4552 if (!isa<BuildVectorSDNode>(V2))
4557 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4565 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4566 for (
unsigned i = S; i != E; ++i)
4567 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4573 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4574 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4577 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4579 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4584 auto OpCode = IsVSlidedown ?
4589 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4592 Splat, TrueMask, VL);
4603 MVT VecContainerVT = VecVT;
4620 MVT WideContainerVT = WideVT;
4626 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4639 }
else if (Subtarget.hasStdExtZvbb()) {
4644 OffsetVec, Passthru, Mask, VL);
4647 Interleaved, EvenV, Passthru, Mask, VL);
4655 Interleaved, OffsetVec, Passthru, Mask, VL);
4663 OddV, Passthru, Mask, VL);
4669 OddV, AllOnesVec, Passthru, Mask, VL);
4677 Interleaved, OddsMul, Passthru, Mask, VL);
4684 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4730 if (ViaEltSize > NumElts)
4739 if (ViaEltSize > NumElts)
4745 if (ViaEltSize > NumElts)
4754 MVT &RotateVT,
unsigned &RotateAmt) {
4760 unsigned NumSubElts;
4762 NumElts, NumSubElts, RotateAmt))
4765 NumElts / NumSubElts);
4827 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4830 OutMasks(VRegsPerSrc, {-1, {}});
4835 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4836 int DstVecIdx = DstIdx / ElemsPerVReg;
4837 int DstSubIdx = DstIdx % ElemsPerVReg;
4838 int SrcIdx = Mask[DstIdx];
4839 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4841 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4842 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4843 if (OutMasks[DstVecIdx].first == -1)
4844 OutMasks[DstVecIdx].first = SrcVecIdx;
4845 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4851 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4852 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4866 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4867 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4868 if (SrcVecIdx == -1)
4870 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4877 unsigned InsertIdx = DstVecIdx * NumOpElts;
4890 MVT VT =
Op.getSimpleValueType();
4905 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
4929 V.getOperand(0).getSimpleValueType().getVectorNumElements();
4930 V = V.getOperand(
Offset / OpElements);
4936 auto *Ld = cast<LoadSDNode>(V);
4946 SDValue Ops[] = {Ld->getChain(),
4964 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
4965 Ld->getPointerInfo().getWithOffset(
Offset),
4966 Ld->getOriginalAlign(),
4970 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
4971 Ld->getOriginalAlign(),
4972 Ld->getMemOperand()->getFlags());
4983 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
4986 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5008 if (Subtarget.hasStdExtZvkb())
5019 LoV = LoSrc == 0 ? V1 : V2;
5023 HiV = HiSrc == 0 ? V1 : V2;
5029 unsigned InvRotate = NumElts - Rotation;
5039 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5059 int EvenSrc, OddSrc;
5064 int Size = Mask.size();
5066 assert(EvenSrc >= 0 &&
"Undef source?");
5067 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5071 assert(OddSrc >= 0 &&
"Undef source?");
5072 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5081 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5090 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5119 MVT IndexContainerVT =
5124 for (
int MaskIndex : Mask) {
5125 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5134 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5148 int MaskIndex = MaskIdx.value();
5149 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
5154 for (
int MaskIndex : Mask) {
5155 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
5162 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5175 for (
int MaskIndex : Mask) {
5176 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5178 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5179 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5181 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5186 std::swap(ShuffleMaskLHS, ShuffleMaskRHS);
5189 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5225RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5227 MVT VT =
Op.getSimpleValueType();
5231 MVT ContainerVT = VT;
5234 if (
Op->isVPOpcode()) {
5235 Mask =
Op.getOperand(1);
5239 VL =
Op.getOperand(2);
5245 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5247 FloatEltVT = MVT::f32;
5254 "Expected legal float type!");
5261 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5264 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5269 if (FloatVT.
bitsGT(VT)) {
5270 if (
Op->isVPOpcode())
5271 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5280 if (!
Op->isVPOpcode())
5284 MVT ContainerFloatVT =
5287 Src, Mask, RTZRM, VL);
5294 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5298 if (
Op->isVPOpcode()) {
5307 else if (IntVT.
bitsGT(VT))
5312 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5317 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5318 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5323 unsigned Adjust = ExponentBias + (EltSize - 1);
5325 if (
Op->isVPOpcode())
5335 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5336 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5347 auto *
Load = cast<LoadSDNode>(
Op);
5348 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5351 Load->getMemoryVT(),
5352 *
Load->getMemOperand()))
5356 MVT VT =
Op.getSimpleValueType();
5358 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5359 "Unexpected unaligned RVV load type");
5363 "Expecting equally-sized RVV vector types to be legal");
5365 Load->getPointerInfo(),
Load->getOriginalAlign(),
5366 Load->getMemOperand()->getFlags());
5376 auto *
Store = cast<StoreSDNode>(
Op);
5377 assert(Store &&
Store->getValue().getValueType().isVector() &&
5378 "Expected vector store");
5381 Store->getMemoryVT(),
5382 *
Store->getMemOperand()))
5389 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5390 "Unexpected unaligned RVV store type");
5394 "Expecting equally-sized RVV vector types to be legal");
5395 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5397 Store->getPointerInfo(),
Store->getOriginalAlign(),
5398 Store->getMemOperand()->getFlags());
5403 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5405 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5432 unsigned ShiftAmt, AddOpc;
5449 if (Subtarget.hasStdExtZtso()) {
5472 "Unexpected custom legalisation");
5493 "Unexpected custom legalisation");
5508 "Unexpected custom legalisation");
5509 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5529 "Unexpected custom legalisation");
5545 MVT VT =
Op.getSimpleValueType();
5547 unsigned Check =
Op.getConstantOperandVal(1);
5548 unsigned TDCMask = 0;
5576 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5581 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5583 VL =
Op.getOperand(3);
5586 VL,
Op->getFlags());
5601 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5603 MVT MaskContainerVT =
5606 VL =
Op.getOperand(3);
5611 Mask, VL,
Op->getFlags());
5614 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5619 DAG.
getUNDEF(ContainerVT), Mask, VL});
5623 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5627 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5631 DAG.
getUNDEF(ContainerVT), Mask, VL});
5647 MVT VT =
Op.getSimpleValueType();
5674 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5681 MVT ContainerVT = VT;
5689 if (
Op->isVPOpcode()) {
5690 Mask =
Op.getOperand(2);
5694 VL =
Op.getOperand(3);
5702 {X, X, DAG.getCondCode(ISD::SETOEQ),
5703 DAG.getUNDEF(ContainerVT), Mask, VL});
5711 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5712 DAG.getUNDEF(ContainerVT), Mask, VL});
5722 DAG.
getUNDEF(ContainerVT), Mask, VL);
5730#define OP_CASE(NODE) \
5732 return RISCVISD::NODE##_VL;
5733#define VP_CASE(NODE) \
5734 case ISD::VP_##NODE: \
5735 return RISCVISD::NODE##_VL;
5737 switch (
Op.getOpcode()) {
5813 case ISD::VP_CTLZ_ZERO_UNDEF:
5816 case ISD::VP_CTTZ_ZERO_UNDEF:
5825 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5830 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5835 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5838 case ISD::VP_SELECT:
5847 case ISD::VP_SIGN_EXTEND:
5849 case ISD::VP_ZERO_EXTEND:
5851 case ISD::VP_FP_TO_SINT:
5853 case ISD::VP_FP_TO_UINT:
5856 case ISD::VP_FMINNUM:
5859 case ISD::VP_FMAXNUM:
5864 case ISD::VP_LLRINT:
5876 "not a RISC-V target specific op");
5882 "adding target specific op should update this function");
5902 "not a RISC-V target specific op");
5908 "adding target specific op should update this function");
5927 if (!
Op.getOperand(j).getValueType().isVector()) {
5928 LoOperands[j] =
Op.getOperand(j);
5929 HiOperands[j] =
Op.getOperand(j);
5932 std::tie(LoOperands[j], HiOperands[j]) =
5937 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5939 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5954 std::tie(LoOperands[j], HiOperands[j]) =
5958 if (!
Op.getOperand(j).getValueType().isVector()) {
5959 LoOperands[j] =
Op.getOperand(j);
5960 HiOperands[j] =
Op.getOperand(j);
5963 std::tie(LoOperands[j], HiOperands[j]) =
5968 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5970 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5980 auto [EVLLo, EVLHi] =
5981 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
5985 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
5987 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6005 if (!
Op.getOperand(j).getValueType().isVector()) {
6006 LoOperands[j] =
Op.getOperand(j);
6007 HiOperands[j] =
Op.getOperand(j);
6010 std::tie(LoOperands[j], HiOperands[j]) =
6015 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6018 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6027 switch (
Op.getOpcode()) {
6033 return lowerGlobalAddress(
Op, DAG);
6035 return lowerBlockAddress(
Op, DAG);
6037 return lowerConstantPool(
Op, DAG);
6039 return lowerJumpTable(
Op, DAG);
6041 return lowerGlobalTLSAddress(
Op, DAG);
6045 return lowerSELECT(
Op, DAG);
6047 return lowerBRCOND(
Op, DAG);
6049 return lowerVASTART(
Op, DAG);
6051 return lowerFRAMEADDR(
Op, DAG);
6053 return lowerRETURNADDR(
Op, DAG);
6060 return lowerShiftLeftParts(
Op, DAG);
6062 return lowerShiftRightParts(
Op, DAG,
true);
6064 return lowerShiftRightParts(
Op, DAG,
false);
6067 if (
Op.getValueType().isFixedLengthVector()) {
6068 assert(Subtarget.hasStdExtZvkb());
6069 return lowerToScalableOp(
Op, DAG);
6071 assert(Subtarget.hasVendorXTHeadBb() &&
6072 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6073 "Unexpected custom legalization");
6075 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6080 EVT VT =
Op.getValueType();
6084 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6090 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6091 Subtarget.hasStdExtZfbfmin()) {
6096 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6103 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6120 "Unexpected types");
6154 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6156 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6158 return LowerINTRINSIC_VOID(
Op, DAG);
6160 return LowerIS_FPCLASS(
Op, DAG);
6162 MVT VT =
Op.getSimpleValueType();
6164 assert(Subtarget.hasStdExtZvbb());
6165 return lowerToScalableOp(
Op, DAG);
6168 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6176 if (!
Op.getSimpleValueType().isVector())
6178 return lowerVectorTruncLike(
Op, DAG);
6181 if (
Op.getOperand(0).getValueType().isVector() &&
6182 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6183 return lowerVectorMaskExt(
Op, DAG, 1);
6186 if (
Op.getOperand(0).getValueType().isVector() &&
6187 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6188 return lowerVectorMaskExt(
Op, DAG, -1);
6191 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6193 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6195 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6197 MVT VT =
Op.getSimpleValueType();
6205 MVT ContainerVT = VT;
6211 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6218 MVT VT =
Op.getSimpleValueType();
6238 }
else if ((Val % 8) == 0) {
6254 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6255 Op.getOperand(1).getValueType() == MVT::i32) {
6267 if (
Op.getValueType() == MVT::nxv32f16 &&
6274 EVT VT =
Op.getValueType();
6277 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6279 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6285 if (!
Op.getValueType().isVector())
6287 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6291 EVT VT =
Op.getValueType();
6294 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6296 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6304 if (!
Op.getValueType().isVector())
6306 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6310 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6313 if (
Op.getValueType().isVector() &&
6314 Op.getValueType().getScalarType() == MVT::f16 &&
6317 if (
Op.getValueType() == MVT::nxv32f16)
6332 Op1.getValueType().isVector() &&
6333 Op1.getValueType().getScalarType() == MVT::f16 &&
6336 if (Op1.getValueType() == MVT::nxv32f16)
6341 Op1.getValueType().getVectorElementCount());
6344 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6354 MVT VT =
Op.getSimpleValueType();
6358 bool IsStrict =
Op->isStrictFPOpcode();
6359 SDValue Src =
Op.getOperand(0 + IsStrict);
6361 MVT SrcVT = Src.getSimpleValueType();
6366 "Unexpected vector element types");
6370 if (EltSize > (2 * SrcEltSize)) {
6382 Op.getOperand(0), Ext);
6386 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6391 auto [FExt, Chain] =
6393 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6400 if (SrcEltSize > (2 * EltSize)) {
6403 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6408 Op.getOperand(0), Src);
6423 Op.getOperand(0), Src);
6437 unsigned RVVOpc = 0;
6438 switch (
Op.getOpcode()) {
6470 "Expected same element count");
6477 Op.getOperand(0), Src, Mask, VL);
6481 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6496 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6503 MVT VT =
Op.getSimpleValueType();
6525 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6541 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6561 return lowerVECREDUCE(
Op, DAG);
6565 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6566 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6567 return lowerVECREDUCE(
Op, DAG);
6574 return lowerFPVECREDUCE(
Op, DAG);
6575 case ISD::VP_REDUCE_ADD:
6576 case ISD::VP_REDUCE_UMAX:
6577 case ISD::VP_REDUCE_SMAX:
6578 case ISD::VP_REDUCE_UMIN:
6579 case ISD::VP_REDUCE_SMIN:
6580 case ISD::VP_REDUCE_FADD:
6581 case ISD::VP_REDUCE_SEQ_FADD:
6582 case ISD::VP_REDUCE_FMIN:
6583 case ISD::VP_REDUCE_FMAX:
6584 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6588 return lowerVPREDUCE(
Op, DAG);
6589 case ISD::VP_REDUCE_AND:
6590 case ISD::VP_REDUCE_OR:
6591 case ISD::VP_REDUCE_XOR:
6592 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6593 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6594 return lowerVPREDUCE(
Op, DAG);
6598 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6601 return lowerINSERT_SUBVECTOR(
Op, DAG);
6603 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6605 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6607 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6609 return lowerSTEP_VECTOR(
Op, DAG);
6611 return lowerVECTOR_REVERSE(
Op, DAG);
6613 return lowerVECTOR_SPLICE(
Op, DAG);
6617 if (
Op.getValueType().getScalarType() == MVT::f16 &&
6620 if (
Op.getValueType() == MVT::nxv32f16)
6632 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6633 return lowerVectorMaskSplat(
Op, DAG);
6641 MVT VT =
Op.getSimpleValueType();
6642 MVT ContainerVT = VT;
6660 Op->ops().take_front(HalfNumOps));
6662 Op->ops().drop_front(HalfNumOps));
6666 unsigned NumOpElts =
6667 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6670 SDValue SubVec = OpIdx.value();
6681 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6683 if (
Op.getValueType().isFixedLengthVector())
6684 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6687 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6689 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6690 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6694 return lowerMaskedLoad(
Op, DAG);
6697 return lowerMaskedStore(
Op, DAG);
6706 EVT VT =
Op.getValueType();
6717 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6719 MVT VT =
Op.getSimpleValueType();
6724 "Unexpected CondCode");
6732 if (isa<ConstantSDNode>(
RHS)) {
6733 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6734 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6753 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6758 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6774 return lowerToScalableOp(
Op, DAG);
6778 if (
Op.getSimpleValueType().isFixedLengthVector())
6779 return lowerToScalableOp(
Op, DAG);
6781 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6782 "Unexpected custom legalisation");
6794 if (
Op.getValueType() == MVT::nxv32f16 &&
6805 return lowerToScalableOp(
Op, DAG);
6808 if (!
Op.getValueType().isVector())
6810 return lowerToScalableOp(
Op, DAG);
6813 if (!
Op.getValueType().isVector())
6815 return lowerToScalableOp(
Op, DAG);
6819 EVT VT =
Op->getValueType(0);
6834 return lowerABS(
Op, DAG);
6839 if (Subtarget.hasStdExtZvbb())
6840 return lowerToScalableOp(
Op, DAG);
6842 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6844 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6846 if (
Op.getValueType() == MVT::nxv32f16 &&
6850 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6857 if (
Op.getValueType() == MVT::nxv32f16 &&
6861 return lowerToScalableOp(
Op, DAG);
6864 return lowerVectorStrictFSetcc(
Op, DAG);
6874 case ISD::VP_GATHER:
6875 return lowerMaskedGather(
Op, DAG);
6877 case ISD::VP_SCATTER:
6878 return lowerMaskedScatter(
Op, DAG);
6880 return lowerGET_ROUNDING(
Op, DAG);
6882 return lowerSET_ROUNDING(
Op, DAG);
6884 return lowerEH_DWARF_CFA(
Op, DAG);
6885 case ISD::VP_SELECT:
6894 case ISD::VP_UADDSAT:
6895 case ISD::VP_USUBSAT:
6896 case ISD::VP_SADDSAT:
6897 case ISD::VP_SSUBSAT:
6899 case ISD::VP_LLRINT:
6900 return lowerVPOp(
Op, DAG);
6904 return lowerLogicVPOp(
Op, DAG);
6913 case ISD::VP_FMINNUM:
6914 case ISD::VP_FMAXNUM:
6915 case ISD::VP_FCOPYSIGN:
6916 if (
Op.getValueType() == MVT::nxv32f16 &&
6924 return lowerVPOp(
Op, DAG);
6925 case ISD::VP_IS_FPCLASS:
6926 return LowerIS_FPCLASS(
Op, DAG);
6927 case ISD::VP_SIGN_EXTEND:
6928 case ISD::VP_ZERO_EXTEND:
6929 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6930 return lowerVPExtMaskOp(
Op, DAG);
6931 return lowerVPOp(
Op, DAG);
6932 case ISD::VP_TRUNCATE:
6933 return lowerVectorTruncLike(
Op, DAG);
6934 case ISD::VP_FP_EXTEND:
6935 case ISD::VP_FP_ROUND:
6936 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6937 case ISD::VP_SINT_TO_FP:
6938 case ISD::VP_UINT_TO_FP:
6939 if (
Op.getValueType().isVector() &&
6940 Op.getValueType().getScalarType() == MVT::f16 &&
6943 if (
Op.getValueType() == MVT::nxv32f16)
6955 case ISD::VP_FP_TO_SINT:
6956 case ISD::VP_FP_TO_UINT:
6958 Op1.getValueType().isVector() &&
6959 Op1.getValueType().getScalarType() == MVT::f16 &&
6962 if (Op1.getValueType() == MVT::nxv32f16)
6967 Op1.getValueType().getVectorElementCount());
6971 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
6973 return lowerVPFPIntConvOp(
Op, DAG);
6975 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6979 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6980 return lowerVPSetCCMaskOp(
Op, DAG);
6986 case ISD::VP_BITREVERSE:
6988 return lowerVPOp(
Op, DAG);
6990 case ISD::VP_CTLZ_ZERO_UNDEF:
6991 if (Subtarget.hasStdExtZvbb())
6992 return lowerVPOp(
Op, DAG);
6993 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6995 case ISD::VP_CTTZ_ZERO_UNDEF:
6996 if (Subtarget.hasStdExtZvbb())
6997 return lowerVPOp(
Op, DAG);
6998 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7000 return lowerVPOp(
Op, DAG);
7001 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7002 return lowerVPStridedLoad(
Op, DAG);
7003 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7004 return lowerVPStridedStore(
Op, DAG);
7006 case ISD::VP_FFLOOR:
7008 case ISD::VP_FNEARBYINT:
7009 case ISD::VP_FROUND:
7010 case ISD::VP_FROUNDEVEN:
7011 case ISD::VP_FROUNDTOZERO:
7012 if (
Op.getValueType() == MVT::nxv32f16 &&
7017 case ISD::VP_FMAXIMUM:
7018 case ISD::VP_FMINIMUM:
7019 if (
Op.getValueType() == MVT::nxv32f16 &&
7024 case ISD::EXPERIMENTAL_VP_SPLICE:
7025 return lowerVPSpliceExperimental(
Op, DAG);
7026 case ISD::EXPERIMENTAL_VP_REVERSE:
7027 return lowerVPReverseExperimental(
Op, DAG);
7045 N->getOffset(), Flags);
7053template <
class NodeTy>
7055 bool IsLocal,
bool IsExternWeak)
const {
7065 if (IsLocal && !Subtarget.allowTaggedGlobals())
7127 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7136 return getAddr(
N, DAG);
7143 return getAddr(
N, DAG);
7150 return getAddr(
N, DAG);
7155 bool UseGOT)
const {
7219 Args.push_back(Entry);
7252 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7266 Addr = getStaticTLSAddr(
N, DAG,
false);
7269 Addr = getStaticTLSAddr(
N, DAG,
true);
7274 : getDynamicTLSAddr(
N, DAG);
7291 if (
LHS == LHS2 &&
RHS == RHS2) {
7296 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7304 return std::nullopt;
7312 MVT VT =
N->getSimpleValueType(0);
7342 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7345 if (~TrueVal == FalseVal) {
7385 if (Subtarget.hasShortForwardBranchOpt())
7388 unsigned SelOpNo = 0;
7398 unsigned ConstSelOpNo = 1;
7399 unsigned OtherSelOpNo = 2;
7400 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7405 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7406 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7410 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7411 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7417 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7419 std::swap(NewConstOps[0], NewConstOps[1]);
7431 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7433 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7436 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7437 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7446 MVT VT =
Op.getSimpleValueType();
7460 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7488 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7492 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7494 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7495 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7497 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7502 DL, VT, LHSVal, CondV);
7518 if (
Op.hasOneUse()) {
7519 unsigned UseOpc =
Op->use_begin()->getOpcode();
7525 return lowerSELECT(NewSel, DAG);
7553 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7574 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7578 if (TrueVal - 1 == FalseVal)
7580 if (TrueVal + 1 == FalseVal)
7587 RHS == TrueV && LHS == FalseV) {
7604 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7630 LHS, RHS, TargetCC,
Op.getOperand(2));
7648 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7660 int XLenInBytes = Subtarget.
getXLen() / 8;
7662 EVT VT =
Op.getValueType();
7665 unsigned Depth =
Op.getConstantOperandVal(0);
7667 int Offset = -(XLenInBytes * 2);
7683 int XLenInBytes = Subtarget.
getXLen() / 8;
7688 EVT VT =
Op.getValueType();
7690 unsigned Depth =
Op.getConstantOperandVal(0);
7692 int Off = -XLenInBytes;
7693 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7712 EVT VT =
Lo.getValueType();
7751 EVT VT =
Lo.getValueType();
7802 MVT VT =
Op.getSimpleValueType();
7828 MVT VecVT =
Op.getSimpleValueType();
7830 "Unexpected SPLAT_VECTOR_PARTS lowering");
7836 MVT ContainerVT = VecVT;
7856 int64_t ExtTrueVal)
const {
7858 MVT VecVT =
Op.getSimpleValueType();
7861 assert(Src.getValueType().isVector() &&
7862 Src.getValueType().getVectorElementType() == MVT::i1);
7883 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7885 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
7888 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
7893SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
7895 MVT ExtVT =
Op.getSimpleValueType();
7899 MVT VT =
Op.getOperand(0).getSimpleValueType();
7925 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7927 EVT MaskVT =
Op.getValueType();
7930 "Unexpected type for vector mask lowering");
7932 MVT VecVT = Src.getSimpleValueType();
7936 VL =
Op.getOperand(2);
7939 MVT ContainerVT = VecVT;
7945 MVT MaskContainerVT =
7952 std::tie(Mask, VL) =
7960 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
7962 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7966 DAG.
getUNDEF(ContainerVT), Mask, VL);
7969 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
7977 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7980 MVT VT =
Op.getSimpleValueType();
7982 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
7986 return lowerVectorMaskTruncLike(
Op, DAG);
7994 MVT SrcVT = Src.getSimpleValueType();
7999 "Unexpected vector truncate lowering");
8001 MVT ContainerVT = SrcVT;
8005 VL =
Op.getOperand(2);
8018 std::tie(Mask, VL) =
8029 }
while (SrcEltVT != DstEltVT);
8038RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8043 MVT VT =
Op.getSimpleValueType();
8044 MVT SrcVT = Src.getSimpleValueType();
8045 MVT ContainerVT = VT;
8066 Chain, Src, Mask, VL);
8067 Chain = Src.getValue(1);
8074 Chain, Src, Mask, VL);
8085RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8088 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8095 MVT VT =
Op.getSimpleValueType();
8097 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8100 MVT SrcVT = Src.getSimpleValueType();
8107 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8110 MVT ContainerVT = VT;
8114 VL =
Op.getOperand(2);
8128 std::tie(Mask, VL) =
8134 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8140 unsigned InterConvOpc =
8145 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8147 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8158static std::optional<MVT>
8164 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8166 if (MaxIdx < MinVLMAX)
8168 else if (MaxIdx < MinVLMAX * 2)
8170 else if (MaxIdx < MinVLMAX * 4)
8175 return std::nullopt;
8188 MVT VecVT =
Op.getSimpleValueType();
8202 MVT ContainerVT = VecVT;
8211 MVT OrigContainerVT = ContainerVT;
8214 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8215 const unsigned OrigIdx = IdxC->getZExtValue();
8218 DL, DAG, Subtarget)) {
8219 ContainerVT = *ShrunkVT;
8228 VLEN && ContainerVT.
bitsGT(M1VT)) {
8231 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8232 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8233 unsigned ExtractIdx =
8252 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8253 const auto *CVal = cast<ConstantSDNode>(Val);
8254 if (isInt<32>(CVal->getSExtValue())) {
8255 IsLegalInsert =
true;
8264 if (IsLegalInsert) {
8270 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8286 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8287 MVT I32ContainerVT =
8298 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8303 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8305 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8310 ValInVec, AlignedIdx);
8320 DAG.
getUNDEF(I32ContainerVT), ValLo,
8321 I32Mask, InsertI64VL);
8323 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8324 I32Mask, InsertI64VL);
8326 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8339 Idx, Mask, InsertVL, Policy);
8343 Slideup, AlignedIdx);
8358 EVT EltVT =
Op.getValueType();
8365 MVT ContainerVT = VecVT;
8381 unsigned WidenVecLen;
8384 unsigned MaxEEW = Subtarget.
getELen();
8389 "the number of elements should be power of 2");
8393 ExtractBitIdx =
Idx;
8395 WideEltVT = LargestEltVT;
8398 ExtractElementIdx = DAG.
getNode(
8409 Vec, ExtractElementIdx);
8425 MVT ContainerVT = VecVT;
8436 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8437 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8439 unsigned OrigIdx = IdxC->getZExtValue();
8442 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8443 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8444 unsigned ExtractIdx =
8454 std::optional<uint64_t> MaxIdx;
8457 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8458 MaxIdx = IdxC->getZExtValue();
8460 if (
auto SmallerVT =
8462 ContainerVT = *SmallerVT;
8509 "Unexpected opcode");
8516 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8521 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8538 if (OpVT.
bitsLT(XLenVT)) {
8545 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8556 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8559 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8570 case Intrinsic::riscv_vslide1up:
8571 case Intrinsic::riscv_vslide1down:
8572 case Intrinsic::riscv_vslide1up_mask:
8573 case Intrinsic::riscv_vslide1down_mask: {
8576 bool IsMasked = NumOps == 7;
8582 std::tie(ScalarLo, ScalarHi) =
8590 if (isa<ConstantSDNode>(AVL)) {
8591 const auto [MinVLMAX, MaxVLMAX] =
8595 if (AVLInt <= MinVLMAX) {
8597 }
else if (AVLInt >= 2 * MaxVLMAX) {
8604 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8638 if (IntNo == Intrinsic::riscv_vslide1up ||
8639 IntNo == Intrinsic::riscv_vslide1up_mask) {
8641 ScalarHi, I32Mask, I32VL);
8643 ScalarLo, I32Mask, I32VL);
8646 ScalarLo, I32Mask, I32VL);
8648 ScalarHi, I32Mask, I32VL);
8697 const unsigned ElementWidth = 8;
8702 [[maybe_unused]]
unsigned MinVF =
8705 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8709 bool Fractional = VF < LMul1VF;
8710 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8731 MVT ContainerVT = OpVT;
8758 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8762 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8779 if (OpVT.
bitsLT(XLenVT)) {
8782 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8795 EVT ValType = V.getValueType();
8796 if (ValType.isVector() && ValType.isFloatingPoint()) {
8799 ValType.getVectorElementCount());
8802 if (ValType.isFixedLengthVector()) {
8804 DAG, V.getSimpleValueType(), Subtarget);
8820 unsigned IntNo =
Op.getConstantOperandVal(0);
8827 case Intrinsic::thread_pointer: {
8831 case Intrinsic::riscv_orc_b:
8832 case Intrinsic::riscv_brev8:
8833 case Intrinsic::riscv_sha256sig0:
8834 case Intrinsic::riscv_sha256sig1:
8835 case Intrinsic::riscv_sha256sum0:
8836 case Intrinsic::riscv_sha256sum1:
8837 case Intrinsic::riscv_sm3p0:
8838 case Intrinsic::riscv_sm3p1: {
8858 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8860 case Intrinsic::riscv_sm4ks:
8861 case Intrinsic::riscv_sm4ed: {
8871 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
8875 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
8878 case Intrinsic::riscv_zip:
8879 case Intrinsic::riscv_unzip: {
8882 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8884 case Intrinsic::riscv_mopr: {
8897 case Intrinsic::riscv_moprr: {
8909 Op.getOperand(2),
Op.getOperand(3));
8911 case Intrinsic::riscv_clmul:
8922 case Intrinsic::riscv_clmulh:
8923 case Intrinsic::riscv_clmulr: {
8941 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
8943 case Intrinsic::experimental_get_vector_length:
8945 case Intrinsic::experimental_cttz_elts:
8947 case Intrinsic::riscv_vmv_x_s: {
8951 case Intrinsic::riscv_vfmv_f_s:
8954 case Intrinsic::riscv_vmv_v_x:
8956 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
8958 case Intrinsic::riscv_vfmv_v_f:
8960 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
8961 case Intrinsic::riscv_vmv_s_x: {
8964 if (
Scalar.getValueType().bitsLE(XLenVT)) {
8967 Op.getOperand(1), Scalar,
Op.getOperand(3));
8970 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
8987 MVT VT =
Op.getSimpleValueType();
8992 if (
Op.getOperand(1).isUndef())
9008 case Intrinsic::riscv_vfmv_s_f:
9010 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9012 case Intrinsic::riscv_vaesdf_vv:
9013 case Intrinsic::riscv_vaesdf_vs:
9014 case Intrinsic::riscv_vaesdm_vv:
9015 case Intrinsic::riscv_vaesdm_vs:
9016 case Intrinsic::riscv_vaesef_vv:
9017 case Intrinsic::riscv_vaesef_vs:
9018 case Intrinsic::riscv_vaesem_vv:
9019 case Intrinsic::riscv_vaesem_vs:
9020 case Intrinsic::riscv_vaeskf1:
9021 case Intrinsic::riscv_vaeskf2:
9022 case Intrinsic::riscv_vaesz_vs:
9023 case Intrinsic::riscv_vsm4k:
9024 case Intrinsic::riscv_vsm4r_vv:
9025 case Intrinsic::riscv_vsm4r_vs: {
9026 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9027 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9028 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9033 case Intrinsic::riscv_vsm3c:
9034 case Intrinsic::riscv_vsm3me: {
9035 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9036 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9041 case Intrinsic::riscv_vsha2ch:
9042 case Intrinsic::riscv_vsha2cl:
9043 case Intrinsic::riscv_vsha2ms: {
9044 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9045 !Subtarget.hasStdExtZvknhb())
9047 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9048 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9049 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9053 case Intrinsic::riscv_sf_vc_v_x:
9054 case Intrinsic::riscv_sf_vc_v_i:
9055 case Intrinsic::riscv_sf_vc_v_xv:
9056 case Intrinsic::riscv_sf_vc_v_iv:
9057 case Intrinsic::riscv_sf_vc_v_vv:
9058 case Intrinsic::riscv_sf_vc_v_fv:
9059 case Intrinsic::riscv_sf_vc_v_xvv:
9060 case Intrinsic::riscv_sf_vc_v_ivv:
9061 case Intrinsic::riscv_sf_vc_v_vvv:
9062 case Intrinsic::riscv_sf_vc_v_fvv:
9063 case Intrinsic::riscv_sf_vc_v_xvw:
9064 case Intrinsic::riscv_sf_vc_v_ivw:
9065 case Intrinsic::riscv_sf_vc_v_vvw:
9066 case Intrinsic::riscv_sf_vc_v_fvw: {
9067 MVT VT =
Op.getSimpleValueType();
9104 MVT VT =
Op.getSimpleValueType();
9108 if (VT.isFloatingPoint()) {
9113 if (VT.isFixedLengthVector())
9123 if (VT.isFixedLengthVector())
9125 if (VT.isFloatingPoint())
9144 unsigned IntNo =
Op.getConstantOperandVal(1);
9148 case Intrinsic::riscv_masked_strided_load: {
9157 MVT VT =
Op->getSimpleValueType(0);
9158 MVT ContainerVT = VT;
9171 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9183 ScalarVT,
Load->getMemOperand());
9189 Load->getMemOperand());
9194 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9199 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9201 Ops.push_back(PassThru);
9203 Ops.push_back(Stride);
9205 Ops.push_back(Mask);
9210 Ops.push_back(Policy);
9216 Load->getMemoryVT(),
Load->getMemOperand());
9217 Chain =
Result.getValue(1);
9223 case Intrinsic::riscv_seg2_load:
9224 case Intrinsic::riscv_seg3_load:
9225 case Intrinsic::riscv_seg4_load:
9226 case Intrinsic::riscv_seg5_load:
9227 case Intrinsic::riscv_seg6_load:
9228 case Intrinsic::riscv_seg7_load:
9229 case Intrinsic::riscv_seg8_load: {
9232 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9233 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9234 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9235 Intrinsic::riscv_vlseg8};
9236 unsigned NF =
Op->getNumValues() - 1;
9237 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9239 MVT VT =
Op->getSimpleValueType(0);
9245 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9247 ContainerVTs.push_back(MVT::Other);
9255 Load->getMemoryVT(),
Load->getMemOperand());
9257 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9263 case Intrinsic::riscv_sf_vc_v_x_se:
9265 case Intrinsic::riscv_sf_vc_v_i_se:
9267 case Intrinsic::riscv_sf_vc_v_xv_se:
9269 case Intrinsic::riscv_sf_vc_v_iv_se:
9271 case Intrinsic::riscv_sf_vc_v_vv_se:
9273 case Intrinsic::riscv_sf_vc_v_fv_se:
9275 case Intrinsic::riscv_sf_vc_v_xvv_se:
9277 case Intrinsic::riscv_sf_vc_v_ivv_se:
9279 case Intrinsic::riscv_sf_vc_v_vvv_se:
9281 case Intrinsic::riscv_sf_vc_v_fvv_se:
9283 case Intrinsic::riscv_sf_vc_v_xvw_se:
9285 case Intrinsic::riscv_sf_vc_v_ivw_se:
9287 case Intrinsic::riscv_sf_vc_v_vvw_se:
9289 case Intrinsic::riscv_sf_vc_v_fvw_se:
9298 unsigned IntNo =
Op.getConstantOperandVal(1);
9302 case Intrinsic::riscv_masked_strided_store: {
9313 MVT ContainerVT = VT;
9327 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9330 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9340 Ops,
Store->getMemoryVT(),
9341 Store->getMemOperand());
9343 case Intrinsic::riscv_seg2_store:
9344 case Intrinsic::riscv_seg3_store:
9345 case Intrinsic::riscv_seg4_store:
9346 case Intrinsic::riscv_seg5_store:
9347 case Intrinsic::riscv_seg6_store:
9348 case Intrinsic::riscv_seg7_store:
9349 case Intrinsic::riscv_seg8_store: {
9352 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9353 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9354 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9355 Intrinsic::riscv_vsseg8};
9358 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9360 MVT VT =
Op->getOperand(2).getSimpleValueType();
9368 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9370 for (
unsigned i = 0; i < NF; i++)
9372 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9377 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9379 case Intrinsic::riscv_sf_vc_xv_se:
9381 case Intrinsic::riscv_sf_vc_iv_se:
9383 case Intrinsic::riscv_sf_vc_vv_se:
9385 case Intrinsic::riscv_sf_vc_fv_se:
9387 case Intrinsic::riscv_sf_vc_xvv_se:
9389 case Intrinsic::riscv_sf_vc_ivv_se:
9391 case Intrinsic::riscv_sf_vc_vvv_se:
9393 case Intrinsic::riscv_sf_vc_fvv_se:
9395 case Intrinsic::riscv_sf_vc_xvw_se:
9397 case Intrinsic::riscv_sf_vc_ivw_se:
9399 case Intrinsic::riscv_sf_vc_vvw_se:
9401 case Intrinsic::riscv_sf_vc_fvw_se:
9409 switch (ISDOpcode) {
9412 case ISD::VP_REDUCE_ADD:
9415 case ISD::VP_REDUCE_UMAX:
9418 case ISD::VP_REDUCE_SMAX:
9421 case ISD::VP_REDUCE_UMIN:
9424 case ISD::VP_REDUCE_SMIN:
9427 case ISD::VP_REDUCE_AND:
9430 case ISD::VP_REDUCE_OR:
9433 case ISD::VP_REDUCE_XOR:
9436 case ISD::VP_REDUCE_FADD:
9438 case ISD::VP_REDUCE_SEQ_FADD:
9440 case ISD::VP_REDUCE_FMAX:
9442 case ISD::VP_REDUCE_FMIN:
9452 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9457 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9458 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9459 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9460 "Unexpected reduction lowering");
9464 MVT ContainerVT = VecVT;
9473 VL =
Op.getOperand(3);
9475 std::tie(Mask, VL) =
9483 switch (
Op.getOpcode()) {
9487 case ISD::VP_REDUCE_AND: {
9497 case ISD::VP_REDUCE_OR:
9504 case ISD::VP_REDUCE_XOR: {
9527 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9531 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9532 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9533 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9534 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9550 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9554 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9557 if (M1VT != InnerVT)
9563 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9582 VecEVT =
Lo.getValueType();
9595 MVT ContainerVT = VecVT;
9615 Mask, VL,
DL, DAG, Subtarget);
9621static std::tuple<unsigned, SDValue, SDValue>
9625 auto Flags =
Op->getFlags();
9626 unsigned Opcode =
Op.getOpcode();
9650 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9658 MVT VecEltVT =
Op.getSimpleValueType();
9662 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9666 MVT ContainerVT = VecVT;
9672 MVT ResVT =
Op.getSimpleValueType();
9675 VL,
DL, DAG, Subtarget);
9680 if (
Op->getFlags().hasNoNaNs())
9686 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9687 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9693 DL, ResVT, NoNaNs, Res,
9720 Vec, Mask, VL,
DL, DAG, Subtarget);
9732 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9741 (OrigIdx != 0 || !Vec.
isUndef())) {
9744 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9747 "Unexpected mask vector lowering");
9780 MVT ContainerVT = VecVT;
9816 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9817 SlideupAmt, Mask, VL, Policy);
9825 unsigned SubRegIdx, RemIdx;
9826 std::tie(SubRegIdx, RemIdx) =
9828 VecVT, SubVecVT, OrigIdx,
TRI);
9847 if (RemIdx == 0 && (!IsSubVecPartReg || Vec.
isUndef()))
9855 MVT InterSubVT = VecVT;
9857 unsigned AlignedIdx = OrigIdx - RemIdx;
9893 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
9894 SlideupAmt, Mask, VL, Policy);
9899 if (VecVT.
bitsGT(InterSubVT))
9905 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
9911 MVT SubVecVT =
Op.getSimpleValueType();
9916 unsigned OrigIdx =
Op.getConstantOperandVal(1);
9927 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9930 "Unexpected mask vector lowering");
9969 MVT ContainerVT = VecVT;
9979 ContainerVT = *ShrunkVT;
9993 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10005 MVT ContainerSubVecVT = SubVecVT;
10009 unsigned SubRegIdx;
10019 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10020 SubRegIdx = Decompose.first;
10022 (OrigIdx % Vscale));
10026 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10027 SubRegIdx = Decompose.first;
10050 MVT InterSubVT = VecVT;
10054 assert(SubRegIdx != RISCV::NoSubRegister);
10068 Vec, SlidedownAmt, Mask, VL);
10077 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10084 MVT VT =
N.getSimpleValueType();
10088 assert(
Op.getSimpleValueType() == VT &&
10089 "Operands and result must be same type");
10093 unsigned NumVals =
N->getNumValues();
10096 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10099 for (
unsigned I = 0;
I < NumVals;
I++) {
10105 if (TruncVals.
size() > 1)
10107 return TruncVals.
front();
10113 MVT VecVT =
Op.getSimpleValueType();
10116 "vector_interleave on non-scalable vector!");
10127 EVT SplitVT = Op0Lo.getValueType();
10130 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10132 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10146 Op.getOperand(0),
Op.getOperand(1));
10173 Concat, EvenIdx, Passthru, Mask, VL);
10175 Concat, OddIdx, Passthru, Mask, VL);
10189 MVT VecVT =
Op.getSimpleValueType();
10192 "vector_interleave on non-scalable vector!");
10205 EVT SplitVT = Op0Lo.getValueType();
10208 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10210 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10232 Op.getOperand(0),
Op.getOperand(1));
10280 MVT VT =
Op.getSimpleValueType();
10285 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10286 if (StepValImm != 1) {
10295 VL, VT,
DL, DAG, Subtarget);
10310 MVT VecVT =
Op.getSimpleValueType();
10320 unsigned MaxVLMAX =
10330 if (MaxVLMAX > 256 && EltSize == 8) {
10377 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10387 MVT VecVT =
Op.getSimpleValueType();
10391 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10392 SDValue DownOffset, UpOffset;
10393 if (ImmValue >= 0) {
10409 DownOffset, TrueMask, UpOffset);
10410 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10416RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10419 auto *
Load = cast<LoadSDNode>(
Op);
10422 Load->getMemoryVT(),
10423 *
Load->getMemOperand()) &&
10424 "Expecting a correctly-aligned load");
10426 MVT VT =
Op.getSimpleValueType();
10432 const auto [MinVLMAX, MaxVLMAX] =
10435 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10449 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10458 Load->getMemoryVT(),
Load->getMemOperand());
10465RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10468 auto *
Store = cast<StoreSDNode>(
Op);
10471 Store->getMemoryVT(),
10472 *
Store->getMemOperand()) &&
10473 "Expecting a correctly-aligned store");
10495 const auto [MinVLMAX, MaxVLMAX] =
10498 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10510 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10513 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10514 Store->getMemoryVT(),
Store->getMemOperand());
10520 MVT VT =
Op.getSimpleValueType();
10522 const auto *MemSD = cast<MemSDNode>(
Op);
10523 EVT MemVT = MemSD->getMemoryVT();
10525 SDValue Chain = MemSD->getChain();
10529 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10530 Mask = VPLoad->getMask();
10532 VL = VPLoad->getVectorLength();
10534 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10535 Mask = MLoad->getMask();
10536 PassThru = MLoad->getPassThru();
10543 MVT ContainerVT = VT;
10557 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10574 Chain =
Result.getValue(1);
10586 const auto *MemSD = cast<MemSDNode>(
Op);
10587 EVT MemVT = MemSD->getMemoryVT();
10589 SDValue Chain = MemSD->getChain();
10593 bool IsCompressingStore =
false;
10594 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10595 Val = VPStore->getValue();
10596 Mask = VPStore->getMask();
10597 VL = VPStore->getVectorLength();
10599 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10600 Val = MStore->getValue();
10601 Mask = MStore->getMask();
10602 IsCompressingStore = MStore->isCompressingStore();
10611 MVT ContainerVT = VT;
10616 if (!IsUnmasked || IsCompressingStore) {
10625 if (IsCompressingStore) {
10628 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10635 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10644 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10648RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10650 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10653 MVT VT =
Op.getSimpleValueType();
10667 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10674 unsigned Opc =
Op.getOpcode();
10681 MVT VT =
Op.getSimpleValueType();
10714 MVT ContainerInVT = InVT;
10733 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10737 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10745 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10750 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10763 MVT VT =
Op.getSimpleValueType();
10767 "Unexpected type for ISD::ABS");
10769 MVT ContainerVT = VT;
10776 if (
Op->getOpcode() == ISD::VP_ABS) {
10777 Mask =
Op->getOperand(1);
10781 VL =
Op->getOperand(2);
10789 DAG.
getUNDEF(ContainerVT), Mask, VL);
10791 DAG.
getUNDEF(ContainerVT), Mask, VL);
10798SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
10801 MVT VT =
Op.getSimpleValueType();
10805 "Can only handle COPYSIGN with matching types.");
10814 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
10819SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
10821 MVT VT =
Op.getSimpleValueType();
10824 MVT I1ContainerVT =
10838 Op2, DAG.
getUNDEF(ContainerVT), VL);
10849 MVT VT =
Op.getSimpleValueType();
10854 for (
const SDValue &V :
Op->op_values()) {
10855 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10858 if (!
V.getValueType().isVector()) {
10864 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
10865 "Only fixed length vectors are supported!");
10879 if (
Op->isStrictFPOpcode()) {
10888 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
10902 MVT VT =
Op.getSimpleValueType();
10905 MVT ContainerVT = VT;
10911 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10917 if (*MaskIdx == OpIdx.index())
10921 if (
Op.getOpcode() == ISD::VP_MERGE) {
10925 assert(
Op.getOpcode() == ISD::VP_SELECT);
10932 if (!
V.getValueType().isFixedLengthVector()) {
10937 MVT OpVT =
V.getSimpleValueType();
10939 assert(useRVVForFixedLengthVectorVT(OpVT) &&
10940 "Only fixed length vectors are supported!");
10945 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
10955 MVT VT =
Op.getSimpleValueType();
10961 MVT ContainerVT = VT;
10971 DAG.
getUNDEF(ContainerVT), Zero, VL);
10974 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
10976 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
10979 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
10988 MVT VT =
Op.getSimpleValueType();
10992 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
10996 MVT ContainerVT = VT;
11006 switch (Condition) {
11074 MVT DstVT =
Op.getSimpleValueType();
11075 MVT SrcVT = Src.getSimpleValueType();
11088 if (DstEltSize >= SrcEltSize) {
11097 if (SrcEltSize == 1) {
11108 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11109 }
else if (DstEltSize > (2 * SrcEltSize)) {
11113 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11119 "Wrong input/output vector types");
11122 if (DstEltSize > (2 * SrcEltSize)) {
11138 MVT InterimFVT = DstVT;
11139 if (SrcEltSize > (2 * DstEltSize)) {
11140 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11147 if (InterimFVT != DstVT) {
11153 "Wrong input/output vector types");
11157 if (DstEltSize == 1) {
11160 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11170 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11180 while (InterimIVT != DstVT) {
11192 MVT VT =
Op.getSimpleValueType();
11199RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11211 MVT VT =
Op.getSimpleValueType();
11212 MVT ContainerVT = VT;
11225 if (IsMaskVector) {
11236 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11245 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11248 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11249 SDValue DownOffset, UpOffset;
11250 if (ImmValue >= 0) {
11264 Op1, DownOffset, Mask, UpOffset);
11268 if (IsMaskVector) {
11272 {Result, DAG.getConstant(0, DL, ContainerVT),
11273 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11283RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11286 MVT VT =
Op.getSimpleValueType();
11293 MVT ContainerVT = VT;
11301 MVT GatherVT = ContainerVT;
11305 if (IsMaskVector) {
11316 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11322 unsigned MaxVLMAX =
11331 if (MaxVLMAX > 256 && EltSize == 8) {
11359 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11361 if (IsMaskVector) {
11384 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11386 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11388 DAG.
getUNDEF(GatherVT), Mask, EVL);
11390 if (IsMaskVector) {
11405 MVT VT =
Op.getSimpleValueType();
11407 return lowerVPOp(
Op, DAG);
11414 MVT ContainerVT = VT;
11433 MVT VT =
Op.getSimpleValueType();
11434 MVT ContainerVT = VT;
11440 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11446 : Intrinsic::riscv_vlse_mask,
11449 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11450 VPNode->getStride()};
11458 Ops.
push_back(VPNode->getVectorLength());
11466 VPNode->getMemoryVT(), VPNode->getMemOperand());
11480 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11481 SDValue StoreVal = VPNode->getValue();
11483 MVT ContainerVT = VT;
11494 : Intrinsic::riscv_vsse_mask,
11497 VPNode->getBasePtr(), VPNode->getStride()};
11505 Ops.
push_back(VPNode->getVectorLength());
11508 Ops, VPNode->getMemoryVT(),
11509 VPNode->getMemOperand());
11521 MVT VT =
Op.getSimpleValueType();
11523 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11524 EVT MemVT = MemSD->getMemoryVT();
11526 SDValue Chain = MemSD->getChain();
11532 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11533 Index = VPGN->getIndex();
11534 Mask = VPGN->getMask();
11536 VL = VPGN->getVectorLength();
11541 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11542 Index = MGN->getIndex();
11543 Mask = MGN->getMask();
11544 PassThru = MGN->getPassThru();
11548 MVT IndexVT =
Index.getSimpleValueType();
11552 "Unexpected VTs!");
11553 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11556 "Unexpected extending MGATHER/VP_GATHER");
11562 MVT ContainerVT = VT;
11586 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11603 Chain =
Result.getValue(1);
11620 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11621 EVT MemVT = MemSD->getMemoryVT();
11623 SDValue Chain = MemSD->getChain();
11626 [[maybe_unused]]
bool IsTruncatingStore =
false;
11629 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11630 Index = VPSN->getIndex();
11631 Mask = VPSN->getMask();
11632 Val = VPSN->getValue();
11633 VL = VPSN->getVectorLength();
11635 IsTruncatingStore =
false;
11638 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11639 Index = MSN->getIndex();
11640 Mask = MSN->getMask();
11641 Val = MSN->getValue();
11642 IsTruncatingStore = MSN->isTruncatingStore();
11646 MVT IndexVT =
Index.getSimpleValueType();
11650 "Unexpected VTs!");
11651 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11654 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11660 MVT ContainerVT = VT;
11684 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11694 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11703 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11711 static const int Table =
11735 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11741 static const unsigned Table =
11764 bool isRISCV64 = Subtarget.
is64Bit();
11828 switch (
N->getOpcode()) {
11830 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
11836 "Unexpected custom legalisation");
11837 bool IsStrict =
N->isStrictFPOpcode();
11840 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
11858 Opc,
DL, VTs, Chain, Op0,
11892 std::tie(Result, Chain) =
11893 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
11921 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
11932 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
11933 "has custom type legalization on riscv32");
11935 SDValue LoCounter, HiCounter;
11939 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
11941 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
11944 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
11946 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
11950 N->getOperand(0), LoCounter, HiCounter);
11974 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
11975 unsigned XLen = Subtarget.
getXLen();
11978 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
11986 if (LHSIsU == RHSIsU)
12003 if (RHSIsU && LHSIsS && !RHSIsS)
12005 else if (LHSIsU && RHSIsS && !LHSIsS)
12015 "Unexpected custom legalisation");
12022 "Unexpected custom legalisation");
12025 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12051 "Unexpected custom legalisation");
12052 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12053 Subtarget.hasVendorXTHeadBb()) &&
12054 "Unexpected custom legalization");
12055 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12056 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12065 "Unexpected custom legalisation");
12079 MVT VT =
N->getSimpleValueType(0);
12080 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12081 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12082 "Unexpected custom legalisation");
12094 if (VT != MVT::i32)
12103 "Unexpected custom legalisation");
12107 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12124 EVT OType =
N->getValueType(1);
12137 "Unexpected custom legalisation");
12154 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12158 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12176 "Unexpected custom legalisation");
12177 if (Subtarget.hasStdExtZbb()) {
12198 "Unexpected custom legalisation");
12204 "Unexpected custom legalisation");
12206 if (Subtarget.hasStdExtZbb()) {
12240 EVT VT =
N->getValueType(0);
12245 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12249 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12250 Subtarget.hasStdExtZfbfmin()) {
12253 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12258 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12260 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12280 MVT VT =
N->getSimpleValueType(0);
12282 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12283 "Unexpected custom legalisation");
12284 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
12310 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12312 "Unexpected EXTRACT_VECTOR_ELT legalization");
12315 MVT ContainerVT = VecVT;
12343 DAG.
getUNDEF(ContainerVT), Mask, VL);
12351 unsigned IntNo =
N->getConstantOperandVal(0);
12355 "Don't know how to custom type legalize this intrinsic!");
12356 case Intrinsic::experimental_get_vector_length: {
12361 case Intrinsic::experimental_cttz_elts: {
12367 case Intrinsic::riscv_orc_b:
12368 case Intrinsic::riscv_brev8:
12369 case Intrinsic::riscv_sha256sig0:
12370 case Intrinsic::riscv_sha256sig1:
12371 case Intrinsic::riscv_sha256sum0:
12372 case Intrinsic::riscv_sha256sum1:
12373 case Intrinsic::riscv_sm3p0:
12374 case Intrinsic::riscv_sm3p1: {
12375 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12395 case Intrinsic::riscv_sm4ks:
12396 case Intrinsic::riscv_sm4ed: {
12404 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12408 case Intrinsic::riscv_mopr: {
12409 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12419 case Intrinsic::riscv_moprr: {
12420 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12432 case Intrinsic::riscv_clmul: {
12433 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12444 case Intrinsic::riscv_clmulh:
12445 case Intrinsic::riscv_clmulr: {
12446 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12474 case Intrinsic::riscv_vmv_x_s: {
12475 EVT VT =
N->getValueType(0);
12477 if (VT.
bitsLT(XLenVT)) {
12486 "Unexpected custom legalization");
12524 case ISD::VP_REDUCE_ADD:
12525 case ISD::VP_REDUCE_AND:
12526 case ISD::VP_REDUCE_OR:
12527 case ISD::VP_REDUCE_XOR:
12528 case ISD::VP_REDUCE_SMAX:
12529 case ISD::VP_REDUCE_UMAX:
12530 case ISD::VP_REDUCE_SMIN:
12531 case ISD::VP_REDUCE_UMIN:
12595 const EVT VT =
N->getValueType(0);
12596 const unsigned Opc =
N->getOpcode();
12603 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12608 "Inconsistent mappings");
12619 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12622 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12637 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12639 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12640 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12644 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12651 if (
LHS.getOpcode() != ReduceOpc)
12666 auto Flags = ReduceVec->
getFlags();
12667 Flags.intersectWith(
N->getFlags());
12668 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12678 auto BinOpToRVVReduce = [](
unsigned Opc) {
12707 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12710 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12713 unsigned Opc =
N->getOpcode();
12714 unsigned ReduceIdx;
12715 if (IsReduction(
N->getOperand(0), Opc))
12717 else if (IsReduction(
N->getOperand(1), Opc))
12723 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12726 SDValue Extract =
N->getOperand(ReduceIdx);
12758 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
12785 if (!Subtarget.hasStdExtZba())
12789 EVT VT =
N->getValueType(0);
12801 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12802 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
12805 int64_t C0 = N0C->getSExtValue();
12806 int64_t C1 = N1C->getSExtValue();
12807 if (C0 <= 0 || C1 <= 0)
12811 int64_t Bits = std::min(C0, C1);
12812 int64_t Diff = std::abs(C0 - C1);
12813 if (Diff != 1 && Diff != 2 && Diff != 3)
12841 EVT VT =
N->getValueType(0);
12849 if ((!Subtarget.hasStdExtZicond() &&
12850 !Subtarget.hasVendorXVentanaCondOps()) ||
12872 bool SwapSelectOps;
12878 SwapSelectOps =
false;
12879 NonConstantVal = FalseVal;
12881 SwapSelectOps =
true;
12882 NonConstantVal = TrueVal;
12888 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
12936 EVT VT =
N->getValueType(0);
12944 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12945 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12951 if (!N0C->hasOneUse())
12953 int64_t C0 = N0C->getSExtValue();
12954 int64_t C1 = N1C->getSExtValue();
12956 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
12959 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
12960 !isInt<12>(C0 * (C1 / C0))) {
12963 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
12964 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
12967 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
12968 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
12995 EVT VT =
N->getValueType(0);
13026 unsigned OuterExtend =
13030 OuterExtend,
SDLoc(
N), VT,
13038 EVT VT =
N->getValueType(0);
13083 EVT VT =
N->getValueType(0);
13087 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13093 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13103 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13125 EVT VT =
N->getValueType(0);
13154 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13178 EVT VT =
N->getValueType(0);
13198 EVT VT =
N->getValueType(0);
13284 EVT VT =
N->getValueType(0);
13355 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13360 const APInt &Imm = ConstN00->getAPIntValue();
13361 if ((Imm + 1).isSignedIntN(12))
13378 LHS.getValueType());
13399 EVT VT =
N->getValueType(0);
13411 if (!Subtarget.hasStdExtZba() && !Subtarget.hasVendorXTHeadBa())
13419 for (
uint64_t Divisor : {3, 5, 9}) {
13420 if (MulAmt % Divisor != 0)
13422 uint64_t MulAmt2 = MulAmt / Divisor;
13429 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13446 if (ScaleShift >= 1 && ScaleShift < 4) {
13447 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13462 for (
uint64_t Divisor : {3, 5, 9}) {
13467 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13479 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13481 if (ScaleShift >= 1 && ScaleShift < 4) {
13482 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13500 EVT VT =
N->getValueType(0);
13508 unsigned AddSubOpc;
13514 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13515 AddSubOpc = V->getOpcode();
13517 SDValue Opnd = V->getOperand(1);
13518 MulOper = V->getOperand(0);
13527 if (IsAddSubWith1(N0)) {
13529 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13532 if (IsAddSubWith1(N1)) {
13534 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13546 if (isIndexTypeSigned(IndexType))
13549 if (!
N->hasOneUse())
13552 EVT VT =
N.getValueType();
13591 EVT SrcVT = Src.getValueType();
13595 NewElen = std::max(NewElen, 8U);
13618 EVT VT =
N->getValueType(0);
13621 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
13625 auto *N1C = dyn_cast<ConstantSDNode>(N1);
13637 if (!isIntEqualitySetCC(
Cond))
13646 const APInt &C1 = N1C->getAPIntValue();
13664 EVT VT =
N->getValueType(0);
13668 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
13670 Src.getOperand(0));
13678struct CombineResult;
13680enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
13707struct NodeExtensionHelper {
13716 bool SupportsFPExt;
13719 bool EnforceOneUse;
13734 return OrigOperand;
13745 unsigned getExtOpc(ExtKind SupportsExt)
const {
13746 switch (SupportsExt) {
13747 case ExtKind::SExt:
13749 case ExtKind::ZExt:
13751 case ExtKind::FPExt:
13762 std::optional<ExtKind> SupportsExt)
const {
13763 if (!SupportsExt.has_value())
13764 return OrigOperand;
13766 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
13770 if (
Source.getValueType() == NarrowVT)
13773 unsigned ExtOpc = getExtOpc(*SupportsExt);
13777 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
13784 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
13802 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
13808 MVT EltVT = SupportsExt == ExtKind::FPExt
13810 :
MVT::getIntegerVT(NarrowSize);
13812 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
13813 "Trying to extend something we can't represent");
13820 static unsigned getSExtOpcode(
unsigned Opcode) {
13843 static unsigned getZExtOpcode(
unsigned Opcode) {
13869 static unsigned getFPExtOpcode(
unsigned Opcode) {
13886 static unsigned getSUOpcode(
unsigned Opcode) {
13888 "SU is only supported for MUL");
13894 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
13914 using CombineToTry = std::function<std::optional<CombineResult>(
13915 SDNode * ,
const NodeExtensionHelper & ,
13920 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
13924 unsigned Opc = OrigOperand.
getOpcode();
13928 "Unexpected Opcode");
13941 unsigned ScalarBits =
Op.getValueSizeInBits();
13944 if (ScalarBits < EltBits)
13950 if (NarrowSize < 8)
13954 SupportsSExt =
true;
13958 SupportsZExt =
true;
13960 EnforceOneUse =
false;
13967 SupportsZExt =
false;
13968 SupportsSExt =
false;
13969 SupportsFPExt =
false;
13970 EnforceOneUse =
true;
13971 unsigned Opc = OrigOperand.
getOpcode();
13993 SupportsZExt =
true;
13996 SupportsSExt =
true;
13999 SupportsFPExt =
true;
14003 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14011 static bool isSupportedRoot(
const SDNode *Root,
14040 Subtarget.hasStdExtZvbb();
14042 return Subtarget.hasStdExtZvbb();
14051 assert(isSupportedRoot(Root, Subtarget) &&
14052 "Trying to build an helper with an "
14053 "unsupported root");
14054 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14070 if (OperandIdx == 1) {
14079 EnforceOneUse =
false;
14084 fillUpExtensionSupport(Root, DAG, Subtarget);
14090 static std::pair<SDValue, SDValue>
14093 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14112 switch (
N->getOpcode()) {
14150struct CombineResult {
14152 unsigned TargetOpcode;
14154 std::optional<ExtKind> LHSExt;
14155 std::optional<ExtKind> RHSExt;
14159 NodeExtensionHelper
LHS;
14161 NodeExtensionHelper
RHS;
14163 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14164 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14165 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14166 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14175 std::tie(Mask, VL) =
14176 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14190 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14191 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14206static std::optional<CombineResult>
14207canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14208 const NodeExtensionHelper &RHS,
14211 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14212 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14213 Root, LHS, {ExtKind::ZExt}, RHS,
14215 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14216 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14217 Root, LHS, {ExtKind::SExt}, RHS,
14219 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14220 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14221 Root, LHS, {ExtKind::FPExt}, RHS,
14223 return std::nullopt;
14232static std::optional<CombineResult>
14233canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14236 return canFoldToVWWithSameExtensionImpl(
14237 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14245static std::optional<CombineResult>
14246canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14249 if (
RHS.SupportsFPExt)
14250 return CombineResult(
14251 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14252 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14259 return CombineResult(
14260 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14261 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14263 return CombineResult(
14264 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14265 LHS, std::nullopt, RHS, {ExtKind::SExt});
14266 return std::nullopt;
14273static std::optional<CombineResult>
14274canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14277 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14285static std::optional<CombineResult>
14286canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14289 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14297static std::optional<CombineResult>
14298canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14301 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14309static std::optional<CombineResult>
14310canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14314 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14315 return std::nullopt;
14316 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14317 Root, LHS, {ExtKind::SExt}, RHS,
14322NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14333 Strategies.
push_back(canFoldToVWWithSameExtension);
14338 Strategies.
push_back(canFoldToVWWithSameExtension);
14343 Strategies.
push_back(canFoldToVWWithSameExtension);
14350 Strategies.
push_back(canFoldToVWWithZEXT);
14355 Strategies.
push_back(canFoldToVWWithSEXT);
14360 Strategies.
push_back(canFoldToVWWithZEXT);
14365 Strategies.
push_back(canFoldToVWWithFPEXT);
14394 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14400 Inserted.insert(
N);
14403 while (!Worklist.
empty()) {
14405 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14408 NodeExtensionHelper
LHS(
N, 0, DAG, Subtarget);
14409 NodeExtensionHelper
RHS(
N, 1, DAG, Subtarget);
14410 auto AppendUsersIfNeeded = [&Worklist,
14411 &Inserted](
const NodeExtensionHelper &
Op) {
14412 if (
Op.needToPromoteOtherUsers()) {
14413 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14414 if (Inserted.insert(TheUse).second)
14426 NodeExtensionHelper::getSupportedFoldings(
N);
14428 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14429 bool Matched =
false;
14430 for (
int Attempt = 0;
14431 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
14434 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14435 FoldingStrategies) {
14436 std::optional<CombineResult> Res =
14437 FoldingStrategy(
N,
LHS,
RHS, DAG, Subtarget);
14444 if (Res->LHSExt.has_value())
14445 AppendUsersIfNeeded(
LHS);
14446 if (Res->RHSExt.has_value())
14447 AppendUsersIfNeeded(
RHS);
14458 SDValue InputRootReplacement;
14465 for (CombineResult Res : CombinesToApply) {
14466 SDValue NewValue = Res.materialize(DAG, Subtarget);
14467 if (!InputRootReplacement) {
14469 "First element is expected to be the current node");
14470 InputRootReplacement = NewValue;
14475 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14479 return InputRootReplacement;
14486 unsigned Opc =
N->getOpcode();
14491 SDValue MergeOp =
N->getOperand(1);
14492 unsigned MergeOpc = MergeOp.
getOpcode();
14503 SDValue Passthru =
N->getOperand(2);
14517 Z = Z.getOperand(1);
14523 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14530 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14559 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14565 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14567 if (MemVT == MVT::i32)
14573 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
14608 if (!Subtarget.hasVendorXTHeadMemPair())
14620 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
14622 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
14623 return {
Ptr->getOperand(0), C1->getZExtValue()};
14627 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
14650 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
14653 if (Base1 != Base2)
14657 bool Valid =
false;
14658 if (MemVT == MVT::i32) {
14660 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
14662 }
else if (MemVT == MVT::i64) {
14664 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
14698 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14706 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14716 EVT VT =
N->getValueType(0);
14719 MVT SrcVT = Src.getSimpleValueType();
14720 MVT SrcContainerVT = SrcVT;
14722 SDValue XVal = Src.getOperand(0);
14749 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14753 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14757 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
14770 if (VT != MVT::i32 && VT != XLenVT)
14800 EVT DstVT =
N->getValueType(0);
14801 if (DstVT != XLenVT)
14807 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14815 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14818 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
14827 if (SatVT == DstVT)
14829 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
14835 Src = Src.getOperand(0);
14856 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
14862 EVT VT =
N->getValueType(0);
14915 unsigned Offset =
N->isTargetStrictFPOpcode();
14922 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
14924 V.getOperand(2) == VL) {
14926 V = V.getOperand(0);
14933 bool NegA = invertIfNegative(
A);
14934 bool NegB = invertIfNegative(
B);
14935 bool NegC = invertIfNegative(
C);
14938 if (!NegA && !NegB && !NegC)
14942 if (
N->isTargetStrictFPOpcode())
14944 {N->getOperand(0), A, B, C, Mask, VL});
14954 if (
N->getValueType(0).isScalableVector() &&
14955 N->getValueType(0).getVectorElementType() == MVT::f32 &&
14962 if (
N->isTargetStrictFPOpcode())
14987 switch (
N->getOpcode()) {
15007 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15008 N->getOperand(2), Mask, VL);
15015 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15018 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15020 uint64_t ShAmt =
N->getConstantOperandVal(1);
15031 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15059 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15072 !isa<ConstantSDNode>(U->getOperand(1)) ||
15073 U->getConstantOperandVal(1) > 32)
15128 if (!
Cond.hasOneUse())
15147 EVT VT =
Cond.getValueType();
15192 LHS =
LHS.getOperand(0);
15202 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15206 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15210 RHS =
LHS.getOperand(1);
15211 LHS =
LHS.getOperand(0);
15220 RHS =
LHS.getOperand(1);
15221 LHS =
LHS.getOperand(0);
15237 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15278 bool Commutative =
true;
15279 unsigned Opc = TrueVal.getOpcode();
15287 Commutative =
false;
15295 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15299 if (FalseVal == TrueVal.getOperand(0))
15301 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15306 EVT VT =
N->getValueType(0);
15308 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15314 assert(IdentityOperand &&
"No identity operand!");
15319 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15320 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15341 CountZeroes =
N->getOperand(2);
15342 ValOnZero =
N->getOperand(1);
15344 CountZeroes =
N->getOperand(1);
15345 ValOnZero =
N->getOperand(2);
15364 if (
Cond->getOperand(0) != CountZeroesArgument)
15380 CountZeroes, BitWidthMinusOne);
15390 EVT VT =
N->getValueType(0);
15391 EVT CondVT =
Cond.getValueType();
15399 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15405 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15426 SDValue TrueVal =
N->getOperand(1);
15427 SDValue FalseVal =
N->getOperand(2);
15442 EVT VT =
N->getValueType(0);
15449 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15464 if (
Op.isUndef()) {
15477 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15481 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15482 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15486 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15514 const unsigned InVecOpcode = InVec->
getOpcode();
15524 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15531 InVecLHS, InValLHS, EltNo);
15533 InVecRHS, InValRHS, EltNo);
15542 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15545 unsigned Elt = IndexC->getZExtValue();
15553 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15556 ConcatOp, InVal, NewIdx);
15560 ConcatOps[ConcatOpIdx] = ConcatOp;
15572 EVT VT =
N->getValueType(0);
15582 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
15584 !
SDValue(BaseLd, 0).hasOneUse())
15587 EVT BaseLdVT = BaseLd->getValueType(0);
15594 auto *Ld = dyn_cast<LoadSDNode>(
Op);
15595 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
15597 Ld->getValueType(0) != BaseLdVT)
15606 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
15608 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
15613 if (BIO1.equalBaseIndex(BIO2, DAG))
15614 return {{BIO2.getOffset() - BIO1.getOffset(),
false}};
15618 SDValue P2 = Ld2->getBasePtr();
15624 return std::nullopt;
15628 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
15633 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
15634 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
15642 unsigned WideScalarBitWidth =
15655 auto [StrideVariant, MustNegateStride] = *BaseDiff;
15656 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
15657 ? std::get<SDValue>(StrideVariant)
15660 if (MustNegateStride)
15673 BaseLd->getBasePtr(), Stride, AllOneMask};
15676 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
15677 ConstStride && ConstStride->getSExtValue() >= 0)
15681 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
15687 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
15691 Ops, WideVecVT, MMO);
15703 if (
N->getValueType(0).isFixedLengthVector())
15706 SDValue Addend =
N->getOperand(0);
15710 SDValue AddMergeOp =
N->getOperand(2);
15715 auto IsVWMulOpc = [](
unsigned Opc) {
15744 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
15745 }(
N, DAG, Subtarget);
15750 if (AddMask != MulMask || AddVL != MulVL)
15755 "Unexpected opcode after VWMACC_VL");
15757 "Unexpected opcode after VWMACC_VL!");
15759 "Unexpected opcode after VWMUL_VL!");
15761 "Unexpected opcode after VWMUL_VL!");
15764 EVT VT =
N->getValueType(0);
15780 const EVT IndexVT =
Index.getValueType();
15784 if (!isIndexTypeSigned(IndexType))
15816 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15819 if (
Index->getOperand(i)->isUndef())
15822 if (
C % ElementSize != 0)
15824 C =
C / ElementSize;
15828 ActiveLanes.
set(
C);
15830 return ActiveLanes.
all();
15848 if (NumElems % 2 != 0)
15852 const unsigned WiderElementSize = ElementSize * 2;
15853 if (WiderElementSize > ST.getELen()/8)
15856 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
15859 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15862 if (
Index->getOperand(i)->isUndef())
15868 if (
C % WiderElementSize != 0)
15873 if (
C !=
Last + ElementSize)
15889 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
15900 switch (
N->getOpcode()) {
15920 APInt V =
C->getValueAPF().bitcastToAPInt();
15955 if (SimplifyDemandedLowBitsHelper(0, 32) ||
15956 SimplifyDemandedLowBitsHelper(1, 5))
15964 if (SimplifyDemandedLowBitsHelper(0, 32))
15981 MVT VT =
N->getSimpleValueType(0);
15990 "Unexpected value type!");
16013 EVT VT =
N->getValueType(0);
16078 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16083 Src.getOperand(0));
16088 Src.getOperand(0), Src.getOperand(1));
16103 auto IsTruncNode = [](
SDValue V) {
16106 SDValue VL = V.getOperand(2);
16107 auto *
C = dyn_cast<ConstantSDNode>(VL);
16109 bool IsVLMAXForVMSET = (
C &&
C->isAllOnes()) ||
16110 (isa<RegisterSDNode>(VL) &&
16111 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16120 while (IsTruncNode(
Op)) {
16121 if (!
Op.hasOneUse())
16123 Op =
Op.getOperand(0);
16154 if (
N->getOperand(1).getOpcode() ==
ISD::XOR &&
16163 N->getOperand(0),
Cond);
16175 SDValue FalseV =
N->getOperand(4);
16177 EVT VT =
N->getValueType(0);
16180 if (TrueV == FalseV)
16185 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16191 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16192 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16195 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16196 isInt<12>(TrueSImm - FalseSImm)) {
16212 {LHS, RHS, CC, TrueV, FalseV});
16279 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16292 EVT VT =
N->getValueType(0);
16316 const auto *MGN = dyn_cast<MaskedGatherSDNode>(
N);
16317 const EVT VT =
N->getValueType(0);
16319 SDValue ScaleOp = MGN->getScale();
16321 assert(!MGN->isIndexScaled() &&
16322 "Scaled gather/scatter should not be formed");
16327 N->getVTList(), MGN->getMemoryVT(),
DL,
16328 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16329 MGN->getBasePtr(), Index, ScaleOp},
16330 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16334 N->getVTList(), MGN->getMemoryVT(),
DL,
16335 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16336 MGN->getBasePtr(), Index, ScaleOp},
16337 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16343 if (std::optional<VIDSequence> SimpleVID =
16345 SimpleVID && SimpleVID->StepDenominator == 1) {
16346 const int64_t StepNumerator = SimpleVID->StepNumerator;
16347 const int64_t Addend = SimpleVID->Addend;
16354 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16364 {MGN->getChain(), IntID, MGN->getPassThru(), BasePtr,
16365 DAG.
getConstant(StepNumerator,
DL, XLenVT), MGN->getMask()};
16367 Ops, VT, MGN->getMemOperand());
16375 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16377 MGN->getMemoryVT(), MGN->getMemOperand(),
16386 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16388 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16390 EVT IndexVT =
Index.getValueType()
16397 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16399 EltCnt.divideCoefficientBy(2));
16402 EltCnt.divideCoefficientBy(2));
16407 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16416 const auto *MSN = dyn_cast<MaskedScatterSDNode>(
N);
16418 SDValue ScaleOp = MSN->getScale();
16420 assert(!MSN->isIndexScaled() &&
16421 "Scaled gather/scatter should not be formed");
16426 N->getVTList(), MSN->getMemoryVT(),
DL,
16427 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16429 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16433 N->getVTList(), MSN->getMemoryVT(),
DL,
16434 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16436 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16438 EVT VT = MSN->getValue()->getValueType(0);
16440 if (!MSN->isTruncatingStore() &&
16444 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
16445 DAG.
getUNDEF(XLenVT), MSN->getMask(),
16446 MSN->getMemoryVT(), MSN->getMemOperand(),
16451 case ISD::VP_GATHER: {
16452 const auto *VPGN = dyn_cast<VPGatherSDNode>(
N);
16454 SDValue ScaleOp = VPGN->getScale();
16456 assert(!VPGN->isIndexScaled() &&
16457 "Scaled gather/scatter should not be formed");
16462 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16463 ScaleOp, VPGN->getMask(),
16464 VPGN->getVectorLength()},
16465 VPGN->getMemOperand(), IndexType);
16469 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16470 ScaleOp, VPGN->getMask(),
16471 VPGN->getVectorLength()},
16472 VPGN->getMemOperand(), IndexType);
16476 case ISD::VP_SCATTER: {
16477 const auto *VPSN = dyn_cast<VPScatterSDNode>(
N);
16479 SDValue ScaleOp = VPSN->getScale();
16481 assert(!VPSN->isIndexScaled() &&
16482 "Scaled gather/scatter should not be formed");
16487 {VPSN->getChain(), VPSN->getValue(),
16488 VPSN->getBasePtr(), Index, ScaleOp,
16489 VPSN->getMask(), VPSN->getVectorLength()},
16490 VPSN->getMemOperand(), IndexType);
16494 {VPSN->getChain(), VPSN->getValue(),
16495 VPSN->getBasePtr(), Index, ScaleOp,
16496 VPSN->getMask(), VPSN->getVectorLength()},
16497 VPSN->getMemOperand(), IndexType);
16511 EVT VT =
N->getValueType(0);
16514 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
16515 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
16533 EVT VT =
N->getValueType(0);
16537 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
16567 if (
N->getValueType(0).isScalableVector() &&
16568 N->getValueType(0).getVectorElementType() == MVT::f32 &&
16583 auto *Store = cast<StoreSDNode>(
N);
16584 SDValue Chain = Store->getChain();
16585 EVT MemVT = Store->getMemoryVT();
16586 SDValue Val = Store->getValue();
16589 bool IsScalarizable =
16591 Store->isSimple() &&
16621 NewVT, *Store->getMemOperand())) {
16623 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
16624 Store->getPointerInfo(), Store->getOriginalAlign(),
16625 Store->getMemOperand()->getFlags());
16633 if (
auto *L = dyn_cast<LoadSDNode>(Val);
16635 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
16637 L->getMemoryVT() == MemVT) {
16640 NewVT, *Store->getMemOperand()) &&
16642 NewVT, *L->getMemOperand())) {
16644 L->getPointerInfo(), L->getOriginalAlign(),
16645 L->getMemOperand()->getFlags());
16646 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
16647 Store->getPointerInfo(), Store->getOriginalAlign(),
16648 Store->getMemOperand()->getFlags());
16660 MVT VecVT = Src.getSimpleValueType();
16667 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
16670 Store->getMemOperand(), Store->getAddressingMode(),
16671 Store->isTruncatingStore(),
false);
16678 EVT VT =
N->getValueType(0);
16700 const MVT VT =
N->getSimpleValueType(0);
16701 SDValue Passthru =
N->getOperand(0);
16702 SDValue Scalar =
N->getOperand(1);
16711 const MVT VT =
N->getSimpleValueType(0);
16712 SDValue Passthru =
N->getOperand(0);
16713 SDValue Scalar =
N->getOperand(1);
16718 unsigned ScalarSize = Scalar.getValueSizeInBits();
16720 if (ScalarSize > EltWidth && Passthru.
isUndef())
16721 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
16728 (!Const || Const->isZero() ||
16729 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
16739 if (
N->getOperand(0).isUndef() &&
16742 Src.getOperand(0).getValueType().isScalableVector()) {
16743 EVT VT =
N->getValueType(0);
16744 EVT SrcVT = Src.getOperand(0).getValueType();
16748 return Src.getOperand(0);
16754 const MVT VT =
N->getSimpleValueType(0);
16755 SDValue Passthru =
N->getOperand(0);
16756 SDValue Scalar =
N->getOperand(1);
16766 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
16776 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
16784 MVT VecVT =
N->getOperand(0).getSimpleValueType();
16786 if (M1VT.
bitsLT(VecVT)) {
16797 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
16802 case Intrinsic::riscv_masked_strided_load: {
16803 MVT VT =
N->getSimpleValueType(0);
16804 auto *Load = cast<MemIntrinsicSDNode>(
N);
16805 SDValue PassThru =
N->getOperand(2);
16807 SDValue Stride =
N->getOperand(4);
16813 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16814 StrideC && StrideC->getZExtValue() == ElementSize)
16816 DAG.
getUNDEF(XLenVT), Mask, PassThru,
16817 Load->getMemoryVT(), Load->getMemOperand(),
16821 case Intrinsic::riscv_masked_strided_store: {
16822 auto *Store = cast<MemIntrinsicSDNode>(
N);
16825 SDValue Stride =
N->getOperand(4);
16830 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
16831 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16832 StrideC && StrideC->getZExtValue() == ElementSize)
16835 Store->getMemoryVT(), Store->getMemOperand(),
16839 case Intrinsic::riscv_vcpop:
16840 case Intrinsic::riscv_vcpop_mask:
16841 case Intrinsic::riscv_vfirst:
16842 case Intrinsic::riscv_vfirst_mask: {
16844 if (IntNo == Intrinsic::riscv_vcpop_mask ||
16845 IntNo == Intrinsic::riscv_vfirst_mask)
16846 VL =
N->getOperand(3);
16851 EVT VT =
N->getValueType(0);
16852 if (IntNo == Intrinsic::riscv_vfirst ||
16853 IntNo == Intrinsic::riscv_vfirst_mask)
16862 EVT VT =
N->getValueType(0);
16866 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
16885 EVT XVT,
unsigned KeptBits)
const {
16890 if (XVT != MVT::i32 && XVT != MVT::i64)
16894 if (KeptBits == 32 || KeptBits == 64)
16898 return Subtarget.hasStdExtZbb() &&
16899 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
16907 "Expected shift op");
16918 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
16919 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
16921 const APInt &C1Int = C1->getAPIntValue();
16922 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
16948 if (C1Cost < ShiftedC1Cost)
16962 EVT VT =
Op.getValueType();
16966 unsigned Opcode =
Op.getOpcode();
16974 const APInt &Mask =
C->getAPIntValue();
16983 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
16984 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
16986 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
16987 if (NewMask == Mask)
16992 Op.getOperand(0), NewC);
17005 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17006 if (IsLegalMask(NewMask))
17007 return UseMask(NewMask);
17010 if (VT == MVT::i64) {
17012 if (IsLegalMask(NewMask))
17013 return UseMask(NewMask);
17028 APInt NewMask = ShrunkMask;
17029 if (MinSignedBits <= 12)
17031 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17037 assert(IsLegalMask(NewMask));
17038 return UseMask(NewMask);
17042 static const uint64_t GREVMasks[] = {
17043 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17044 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17046 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17047 unsigned Shift = 1 << Stage;
17048 if (ShAmt & Shift) {
17050 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17062 const APInt &DemandedElts,
17064 unsigned Depth)
const {
17066 unsigned Opc =
Op.getOpcode();
17071 "Should use MaskedValueIsZero if you don't know whether Op"
17072 " is a target node!");
17155 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17158 if (MaxVLenB == MinVLenB)
17175 case Intrinsic::riscv_vsetvli:
17176 case Intrinsic::riscv_vsetvlimax: {
17177 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17178 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17184 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17187 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17188 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17190 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17203 unsigned Depth)
const {
17204 switch (
Op.getOpcode()) {
17210 if (Tmp == 1)
return 1;
17213 return std::min(Tmp, Tmp2);
17225 if (Tmp < 33)
return 1;
17250 unsigned XLen = Subtarget.
getXLen();
17251 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17252 if (EltBits <= XLen)
17253 return XLen - EltBits + 1;
17257 unsigned IntNo =
Op.getConstantOperandVal(1);
17261 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17262 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17263 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17264 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17265 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17266 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17267 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17268 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17269 case Intrinsic::riscv_masked_cmpxchg_i64:
17277 assert(Subtarget.hasStdExtA());
17292 switch (
Op.getOpcode()) {
17298 return !
Op.getValueType().isInteger();
17306 assert(Ld &&
"Unexpected null LoadSDNode");
17314 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17315 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17316 CNode->getOffset() != 0)
17324 auto *CNode = GetSupportedConstantPool(
Ptr);
17325 if (!CNode || CNode->getTargetFlags() != 0)
17328 return CNode->getConstVal();
17336 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17337 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17343 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17346 return CNodeLo->getConstVal();
17351 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17383 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17386 int64_t LoCounter =
MI.getOperand(2).getImm();
17387 int64_t HiCounter =
MI.getOperand(3).getImm();
17397 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17409 MI.eraseFromParent();
17417 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
17425 Register SrcReg =
MI.getOperand(2).getReg();
17445 MI.eraseFromParent();
17452 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
17453 "Unexpected instruction");
17459 Register DstReg =
MI.getOperand(0).getReg();
17482 MI.eraseFromParent();
17487 switch (
MI.getOpcode()) {
17490 case RISCV::Select_GPR_Using_CC_GPR:
17491 case RISCV::Select_FPR16_Using_CC_GPR:
17492 case RISCV::Select_FPR16INX_Using_CC_GPR:
17493 case RISCV::Select_FPR32_Using_CC_GPR:
17494 case RISCV::Select_FPR32INX_Using_CC_GPR:
17495 case RISCV::Select_FPR64_Using_CC_GPR:
17496 case RISCV::Select_FPR64INX_Using_CC_GPR:
17497 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
17503 unsigned RelOpcode,
unsigned EqOpcode,
17506 Register DstReg =
MI.getOperand(0).getReg();
17507 Register Src1Reg =
MI.getOperand(1).getReg();
17508 Register Src2Reg =
MI.getOperand(2).getReg();
17510 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17534 MI.eraseFromParent();
17585 F->insert(It, FirstMBB);
17586 F->insert(It, SecondMBB);
17587 F->insert(It, SinkMBB);
17636 First.eraseFromParent();
17679 SelectDests.
insert(
MI.getOperand(0).getReg());
17683 if (
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR && Next != BB->
end() &&
17684 Next->getOpcode() ==
MI.getOpcode() &&
17685 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
17686 Next->getOperand(5).isKill()) {
17691 SequenceMBBI != E; ++SequenceMBBI) {
17692 if (SequenceMBBI->isDebugInstr())
17695 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
17696 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
17697 SequenceMBBI->getOperand(3).getImm() !=
CC ||
17698 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
17699 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
17701 LastSelectPseudo = &*SequenceMBBI;
17703 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
17706 if (SequenceMBBI->hasUnmodeledSideEffects() ||
17707 SequenceMBBI->mayLoadOrStore() ||
17708 SequenceMBBI->usesCustomInsertionHook())
17711 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
17726 F->insert(
I, IfFalseMBB);
17727 F->insert(
I, TailMBB);
17731 TailMBB->
push_back(DebugInstr->removeFromParent());
17735 TailMBB->
splice(TailMBB->
end(), HeadMBB,
17754 auto SelectMBBI =
MI.getIterator();
17755 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
17756 auto InsertionPoint = TailMBB->
begin();
17757 while (SelectMBBI != SelectEnd) {
17758 auto Next = std::next(SelectMBBI);
17761 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
17762 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
17763 .
addReg(SelectMBBI->getOperand(4).getReg())
17765 .
addReg(SelectMBBI->getOperand(5).getReg())
17778 unsigned CVTXOpc) {
17784 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17797 .
add(
MI.getOperand(1))
17798 .
add(
MI.getOperand(2))
17799 .
add(
MI.getOperand(3))
17801 .
add(
MI.getOperand(4))
17802 .
add(
MI.getOperand(5))
17803 .
add(
MI.getOperand(6))
17818 if (Log2SEW == 4) {
17821 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M1_E16_MASK;
17824 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M2_E16_MASK;
17827 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M4_E16_MASK;
17830 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M8_E16_MASK;
17833 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_MF2_E16_MASK;
17836 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_MF4_E16_MASK;
17842 }
else if (Log2SEW == 5) {
17845 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M1_E32_MASK;
17848 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M2_E32_MASK;
17851 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M4_E32_MASK;
17854 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M8_E32_MASK;
17857 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_MF2_E32_MASK;
17864 }
else if (Log2SEW == 6) {
17867 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M1_E64_MASK;
17870 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M2_E64_MASK;
17873 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M4_E64_MASK;
17876 CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M8_E64_MASK;
17889 .
add(
MI.getOperand(0))
17890 .
add(
MI.getOperand(1))
17892 .
add(
MI.getOperand(3))
17894 .
add(
MI.getOperand(4))
17895 .
add(
MI.getOperand(5))
17896 .
add(
MI.getOperand(6))
17906 MI.eraseFromParent();
17912 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
17914 switch (
MI.getOpcode()) {
17917 case RISCV::PseudoFROUND_H:
17918 CmpOpc = RISCV::FLT_H;
17919 F2IOpc = RISCV::FCVT_W_H;
17920 I2FOpc = RISCV::FCVT_H_W;
17921 FSGNJOpc = RISCV::FSGNJ_H;
17922 FSGNJXOpc = RISCV::FSGNJX_H;
17923 RC = &RISCV::FPR16RegClass;
17925 case RISCV::PseudoFROUND_H_INX:
17926 CmpOpc = RISCV::FLT_H_INX;
17927 F2IOpc = RISCV::FCVT_W_H_INX;
17928 I2FOpc = RISCV::FCVT_H_W_INX;
17929 FSGNJOpc = RISCV::FSGNJ_H_INX;
17930 FSGNJXOpc = RISCV::FSGNJX_H_INX;
17931 RC = &RISCV::GPRF16RegClass;
17933 case RISCV::PseudoFROUND_S:
17934 CmpOpc = RISCV::FLT_S;
17935 F2IOpc = RISCV::FCVT_W_S;
17936 I2FOpc = RISCV::FCVT_S_W;
17937 FSGNJOpc = RISCV::FSGNJ_S;
17938 FSGNJXOpc = RISCV::FSGNJX_S;
17939 RC = &RISCV::FPR32RegClass;
17941 case RISCV::PseudoFROUND_S_INX:
17942 CmpOpc = RISCV::FLT_S_INX;
17943 F2IOpc = RISCV::FCVT_W_S_INX;
17944 I2FOpc = RISCV::FCVT_S_W_INX;
17945 FSGNJOpc = RISCV::FSGNJ_S_INX;
17946 FSGNJXOpc = RISCV::FSGNJX_S_INX;
17947 RC = &RISCV::GPRF32RegClass;
17949 case RISCV::PseudoFROUND_D:
17951 CmpOpc = RISCV::FLT_D;
17952 F2IOpc = RISCV::FCVT_L_D;
17953 I2FOpc = RISCV::FCVT_D_L;
17954 FSGNJOpc = RISCV::FSGNJ_D;
17955 FSGNJXOpc = RISCV::FSGNJX_D;
17956 RC = &RISCV::FPR64RegClass;
17958 case RISCV::PseudoFROUND_D_INX:
17960 CmpOpc = RISCV::FLT_D_INX;
17961 F2IOpc = RISCV::FCVT_L_D_INX;
17962 I2FOpc = RISCV::FCVT_D_L_INX;
17963 FSGNJOpc = RISCV::FSGNJ_D_INX;
17964 FSGNJXOpc = RISCV::FSGNJX_D_INX;
17965 RC = &RISCV::GPRRegClass;
17977 F->insert(
I, CvtMBB);
17978 F->insert(
I, DoneMBB);
17989 Register DstReg =
MI.getOperand(0).getReg();
17990 Register SrcReg =
MI.getOperand(1).getReg();
17991 Register MaxReg =
MI.getOperand(2).getReg();
17992 int64_t FRM =
MI.getOperand(3).getImm();
17997 Register FabsReg =
MRI.createVirtualRegister(RC);
18001 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18016 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18038 MI.eraseFromParent();
18045 switch (
MI.getOpcode()) {
18048 case RISCV::ReadCounterWide:
18050 "ReadCounterWide is only to be used on riscv32");
18052 case RISCV::Select_GPR_Using_CC_GPR:
18053 case RISCV::Select_FPR16_Using_CC_GPR:
18054 case RISCV::Select_FPR16INX_Using_CC_GPR:
18055 case RISCV::Select_FPR32_Using_CC_GPR:
18056 case RISCV::Select_FPR32INX_Using_CC_GPR:
18057 case RISCV::Select_FPR64_Using_CC_GPR:
18058 case RISCV::Select_FPR64INX_Using_CC_GPR:
18059 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18061 case RISCV::BuildPairF64Pseudo:
18063 case RISCV::SplitF64Pseudo:
18065 case RISCV::PseudoQuietFLE_H:
18067 case RISCV::PseudoQuietFLE_H_INX:
18068 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18069 case RISCV::PseudoQuietFLT_H:
18071 case RISCV::PseudoQuietFLT_H_INX:
18072 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18073 case RISCV::PseudoQuietFLE_S:
18075 case RISCV::PseudoQuietFLE_S_INX:
18076 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18077 case RISCV::PseudoQuietFLT_S:
18079 case RISCV::PseudoQuietFLT_S_INX:
18080 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18081 case RISCV::PseudoQuietFLE_D:
18083 case RISCV::PseudoQuietFLE_D_INX:
18084 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18085 case RISCV::PseudoQuietFLE_D_IN32X:
18088 case RISCV::PseudoQuietFLT_D:
18090 case RISCV::PseudoQuietFLT_D_INX:
18091 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18092 case RISCV::PseudoQuietFLT_D_IN32X:
18096 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18098 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18100 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18102 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18104 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18106 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18108 case RISCV::PseudoFROUND_H:
18109 case RISCV::PseudoFROUND_H_INX:
18110 case RISCV::PseudoFROUND_S:
18111 case RISCV::PseudoFROUND_S_INX:
18112 case RISCV::PseudoFROUND_D:
18113 case RISCV::PseudoFROUND_D_INX:
18114 case RISCV::PseudoFROUND_D_IN32X:
18116 case TargetOpcode::STATEPOINT:
18122 MI.addOperand(*
MI.getMF(),
18128 case TargetOpcode::STACKMAP:
18129 case TargetOpcode::PATCHPOINT:
18132 "supported on 64-bit targets");
18150 if (
MI.readsRegister(RISCV::FRM))
18182 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18183 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18186 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18187 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18190 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18191 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18195 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18196 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18197 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18199 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18200 RISCV::V20M2, RISCV::V22M2};
18208 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18209 RISCV::X13, RISCV::X14, RISCV::X15,
18210 RISCV::X16, RISCV::X17};
18212 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18213 RISCV::X13, RISCV::X14, RISCV::X15};
18224 static const MCPhysReg FastCCIGPRs[] = {
18225 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
18226 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
18227 RISCV::X29, RISCV::X30, RISCV::X31};
18230 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18231 RISCV::X13, RISCV::X14, RISCV::X15,
18246 unsigned XLenInBytes = XLen / 8;
18259 Align StackAlign(XLenInBytes);
18260 if (!
EABI || XLen != 32)
18292 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18293 assert(XLen == 32 || XLen == 64);
18294 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18298 if (ArgFlags.
isNest()) {
18299 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18307 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18312 bool UseGPRForF16_F32 =
true;
18315 bool UseGPRForF64 =
true;
18327 UseGPRForF16_F32 = !IsFixed;
18331 UseGPRForF16_F32 = !IsFixed;
18332 UseGPRForF64 = !IsFixed;
18338 UseGPRForF16_F32 =
true;
18339 UseGPRForF64 =
true;
18346 if (UseGPRForF16_F32 &&
18347 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18350 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18367 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18369 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18373 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18382 "PendingLocs and PendingArgFlags out of sync");
18386 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18387 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18435 PendingLocs.
size() <= 2) {
18436 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18441 PendingLocs.
clear();
18442 PendingArgFlags.
clear();
18444 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18450 unsigned StoreSizeBytes = XLen / 8;
18453 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18455 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18457 else if (ValVT == MVT::f64 && !UseGPRForF64)
18493 if (!PendingLocs.
empty()) {
18495 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
18497 for (
auto &It : PendingLocs) {
18499 It.convertToReg(Reg);
18504 PendingLocs.clear();
18505 PendingArgFlags.
clear();
18509 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
18511 "Expected an XLenVT or vector types at this stage");
18529template <
typename ArgTy>
18531 for (
const auto &ArgIdx :
enumerate(Args)) {
18532 MVT ArgVT = ArgIdx.value().VT;
18534 return ArgIdx.index();
18536 return std::nullopt;
18539void RISCVTargetLowering::analyzeInputArgs(
18542 RISCVCCAssignFn Fn)
const {
18543 unsigned NumArgs =
Ins.size();
18556 for (
unsigned i = 0; i != NumArgs; ++i) {
18560 Type *ArgTy =
nullptr;
18562 ArgTy = FType->getReturnType();
18563 else if (Ins[i].isOrigArg())
18564 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
18568 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
18570 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
18577void RISCVTargetLowering::analyzeOutputArgs(
18580 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
18581 unsigned NumArgs = Outs.
size();
18591 for (
unsigned i = 0; i != NumArgs; i++) {
18592 MVT ArgVT = Outs[i].VT;
18594 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
18598 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
18600 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
18654 if (In.isOrigArg()) {
18659 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
18660 (
BitWidth < 32 && In.Flags.isZExt())) {
18689 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
18735 ExtType,
DL, LocVT, Chain, FIN,
18752 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18765 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18775 unsigned ValNo,
MVT ValVT,
MVT LocVT,
18778 bool IsFixed,
bool IsRet,
Type *OrigTy,
18781 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18790 if (LocVT == MVT::f16 &&
18791 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
18793 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
18794 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
18795 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
18796 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
18797 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
18803 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18805 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
18806 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
18807 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
18808 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
18809 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18815 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18817 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
18818 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
18819 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
18820 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
18821 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18828 if ((LocVT == MVT::f16 &&
18829 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
18830 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18831 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
18832 Subtarget.hasStdExtZdinx())) {
18839 if (LocVT == MVT::f16) {
18845 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
18851 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
18859 if (AllocatedVReg) {
18894 if (ArgFlags.
isNest()) {
18896 "Attribute 'nest' is not supported in GHC calling convention");
18900 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
18901 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
18903 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18915 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18918 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
18919 RISCV::F18_F, RISCV::F19_F,
18920 RISCV::F20_F, RISCV::F21_F};
18921 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18927 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18930 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
18931 RISCV::F24_D, RISCV::F25_D,
18932 RISCV::F26_D, RISCV::F27_D};
18933 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18939 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18940 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
18960 switch (CallConv) {
18970 if (Subtarget.hasStdExtE())
18974 "(Zdinx/D) instruction set extensions");
18978 if (Func.hasFnAttribute(
"interrupt")) {
18979 if (!Func.arg_empty())
18981 "Functions with the interrupt attribute cannot have arguments!");
18986 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
18988 "Function interrupt attribute argument not supported!");
18993 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
18995 std::vector<SDValue> OutChains;
19004 analyzeInputArgs(MF, CCInfo, Ins,
false,
19008 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19029 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19030 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19032 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19034 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19063 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19068 if (VarArgsSaveSize == 0) {
19072 int VaArgOffset = -VarArgsSaveSize;
19080 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19081 VarArgsSaveSize += XLenInBytes;
19088 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19093 Chain,
DL, ArgValue, FIN,
19095 OutChains.push_back(Store);
19109 if (!OutChains.empty()) {
19110 OutChains.push_back(Chain);
19120bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19124 auto CalleeCC = CLI.CallConv;
19125 auto &Outs = CLI.Outs;
19127 auto CallerCC = Caller.getCallingConv();
19134 if (Caller.hasFnAttribute(
"interrupt"))
19149 for (
auto &VA : ArgLocs)
19155 auto IsCallerStructRet = Caller.hasStructRetAttr();
19156 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19157 if (IsCallerStructRet || IsCalleeStructRet)
19162 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19163 if (CalleeCC != CallerCC) {
19164 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19165 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19172 for (
auto &Arg : Outs)
19173 if (Arg.Flags.isByVal())
19208 if (Subtarget.hasStdExtE())
19212 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19218 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19224 "site marked musttail");
19231 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19233 if (!Flags.isByVal())
19237 unsigned Size = Flags.getByValSize();
19238 Align Alignment = Flags.getNonZeroByValAlign();
19245 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19259 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19262 SDValue ArgValue = OutVals[OutIdx];
19282 if (!StackPtr.getNode())
19293 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19311 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19312 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19318 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19319 SDValue PartValue = OutVals[OutIdx + 1];
19320 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19332 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19334 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19336 for (
const auto &Part : Parts) {
19337 SDValue PartValue = Part.first;
19338 SDValue PartOffset = Part.second;
19345 ArgValue = SpillSlot;
19351 if (Flags.isByVal())
19352 ArgValue = ByValArgs[j++];
19359 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19360 "for passing parameters");
19363 if (!StackPtr.getNode())
19376 if (!MemOpChains.
empty())
19382 for (
auto &Reg : RegsToPass) {
19383 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19390 validateCCReservedRegs(RegsToPass, MF);
19395 "Return address register required, but has been reserved."});
19414 for (
auto &Reg : RegsToPass)
19420 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19421 assert(Mask &&
"Missing call preserved mask for calling convention");
19430 "Unexpected CFI type for a direct call");
19460 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19461 auto &VA = RVLocs[i];
19469 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19470 assert(VA.needsCustom());
19495 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19496 MVT VT = Outs[i].VT;
19500 ArgFlags, CCInfo,
true,
true,
19501 nullptr, *
this, Dispatcher))
19533 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19534 SDValue Val = OutVals[OutIdx];
19543 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19547 Register RegHi = RVLocs[++i].getLocReg();
19553 "Return value register required, but has been reserved."});
19569 "Return value register required, but has been reserved."});
19591 if (Func.hasFnAttribute(
"interrupt")) {
19592 if (!Func.getReturnType()->isVoidTy())
19594 "Functions with the interrupt attribute must have void return type!");
19600 if (Kind ==
"supervisor")
19606 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19609void RISCVTargetLowering::validateCCReservedRegs(
19610 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19619 F,
"Argument register required, but has been reserved."});
19625 if (
N->getNumValues() != 1)
19627 if (!
N->hasNUsesOfValue(1, 0))
19630 SDNode *Copy = *
N->use_begin();
19644 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
19648 bool HasRet =
false;
19649 for (
SDNode *Node : Copy->uses()) {
19657 Chain = Copy->getOperand(0);
19666#define NODE_NAME_CASE(NODE) \
19667 case RISCVISD::NODE: \
19668 return "RISCVISD::" #NODE;
19921#undef NODE_NAME_CASE
19928 if (Constraint.
size() == 1) {
19929 switch (Constraint[0]) {
19945 if (Constraint ==
"vr" || Constraint ==
"vm")
19951std::pair<unsigned, const TargetRegisterClass *>
19957 if (Constraint.
size() == 1) {
19958 switch (Constraint[0]) {
19963 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
19964 return std::make_pair(0U, &RISCV::GPRF16RegClass);
19965 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
19966 return std::make_pair(0U, &RISCV::GPRF32RegClass);
19967 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
19968 return std::make_pair(0U, &RISCV::GPRPairRegClass);
19969 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
19971 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
19972 return std::make_pair(0U, &RISCV::FPR16RegClass);
19973 if (Subtarget.hasStdExtF() && VT == MVT::f32)
19974 return std::make_pair(0U, &RISCV::FPR32RegClass);
19975 if (Subtarget.hasStdExtD() && VT == MVT::f64)
19976 return std::make_pair(0U, &RISCV::FPR64RegClass);
19981 }
else if (Constraint ==
"vr") {
19982 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
19983 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
19985 return std::make_pair(0U, RC);
19987 }
else if (Constraint ==
"vm") {
19988 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
19989 return std::make_pair(0U, &RISCV::VMV0RegClass);
19997 .
Case(
"{zero}", RISCV::X0)
19998 .
Case(
"{ra}", RISCV::X1)
19999 .
Case(
"{sp}", RISCV::X2)
20000 .
Case(
"{gp}", RISCV::X3)
20001 .
Case(
"{tp}", RISCV::X4)
20002 .
Case(
"{t0}", RISCV::X5)
20003 .
Case(
"{t1}", RISCV::X6)
20004 .
Case(
"{t2}", RISCV::X7)
20005 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20006 .
Case(
"{s1}", RISCV::X9)
20007 .
Case(
"{a0}", RISCV::X10)
20008 .
Case(
"{a1}", RISCV::X11)
20009 .
Case(
"{a2}", RISCV::X12)
20010 .
Case(
"{a3}", RISCV::X13)
20011 .
Case(
"{a4}", RISCV::X14)
20012 .
Case(
"{a5}", RISCV::X15)
20013 .
Case(
"{a6}", RISCV::X16)
20014 .
Case(
"{a7}", RISCV::X17)
20015 .
Case(
"{s2}", RISCV::X18)
20016 .
Case(
"{s3}", RISCV::X19)
20017 .
Case(
"{s4}", RISCV::X20)
20018 .
Case(
"{s5}", RISCV::X21)
20019 .
Case(
"{s6}", RISCV::X22)
20020 .
Case(
"{s7}", RISCV::X23)
20021 .
Case(
"{s8}", RISCV::X24)
20022 .
Case(
"{s9}", RISCV::X25)
20023 .
Case(
"{s10}", RISCV::X26)
20024 .
Case(
"{s11}", RISCV::X27)
20025 .
Case(
"{t3}", RISCV::X28)
20026 .
Case(
"{t4}", RISCV::X29)
20027 .
Case(
"{t5}", RISCV::X30)
20028 .
Case(
"{t6}", RISCV::X31)
20030 if (XRegFromAlias != RISCV::NoRegister)
20031 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20040 if (Subtarget.hasStdExtF()) {
20042 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20043 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20044 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20045 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20046 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20047 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20048 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20049 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20050 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20051 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20052 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20053 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20054 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20055 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20056 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20057 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20058 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20059 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20060 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20061 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20062 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20063 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20064 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20065 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20066 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20067 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20068 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20069 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20070 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20071 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20072 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20073 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20075 if (FReg != RISCV::NoRegister) {
20076 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20077 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20078 unsigned RegNo = FReg - RISCV::F0_F;
20079 unsigned DReg = RISCV::F0_D + RegNo;
20080 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20082 if (VT == MVT::f32 || VT == MVT::Other)
20083 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20084 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20085 unsigned RegNo = FReg - RISCV::F0_F;
20086 unsigned HReg = RISCV::F0_H + RegNo;
20087 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20094 .
Case(
"{v0}", RISCV::V0)
20095 .
Case(
"{v1}", RISCV::V1)
20096 .
Case(
"{v2}", RISCV::V2)
20097 .
Case(
"{v3}", RISCV::V3)
20098 .
Case(
"{v4}", RISCV::V4)
20099 .
Case(
"{v5}", RISCV::V5)
20100 .
Case(
"{v6}", RISCV::V6)
20101 .
Case(
"{v7}", RISCV::V7)
20102 .
Case(
"{v8}", RISCV::V8)
20103 .
Case(
"{v9}", RISCV::V9)
20104 .
Case(
"{v10}", RISCV::V10)
20105 .
Case(
"{v11}", RISCV::V11)
20106 .
Case(
"{v12}", RISCV::V12)
20107 .
Case(
"{v13}", RISCV::V13)
20108 .
Case(
"{v14}", RISCV::V14)
20109 .
Case(
"{v15}", RISCV::V15)
20110 .
Case(
"{v16}", RISCV::V16)
20111 .
Case(
"{v17}", RISCV::V17)
20112 .
Case(
"{v18}", RISCV::V18)
20113 .
Case(
"{v19}", RISCV::V19)
20114 .
Case(
"{v20}", RISCV::V20)
20115 .
Case(
"{v21}", RISCV::V21)
20116 .
Case(
"{v22}", RISCV::V22)
20117 .
Case(
"{v23}", RISCV::V23)
20118 .
Case(
"{v24}", RISCV::V24)
20119 .
Case(
"{v25}", RISCV::V25)
20120 .
Case(
"{v26}", RISCV::V26)
20121 .
Case(
"{v27}", RISCV::V27)
20122 .
Case(
"{v28}", RISCV::V28)
20123 .
Case(
"{v29}", RISCV::V29)
20124 .
Case(
"{v30}", RISCV::V30)
20125 .
Case(
"{v31}", RISCV::V31)
20127 if (VReg != RISCV::NoRegister) {
20128 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20129 return std::make_pair(VReg, &RISCV::VMRegClass);
20130 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20131 return std::make_pair(VReg, &RISCV::VRRegClass);
20132 for (
const auto *RC :
20133 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20134 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20135 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20136 return std::make_pair(VReg, RC);
20142 std::pair<Register, const TargetRegisterClass *> Res =
20148 if (Res.second == &RISCV::GPRF16RegClass ||
20149 Res.second == &RISCV::GPRF32RegClass ||
20150 Res.second == &RISCV::GPRPairRegClass)
20151 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20159 if (ConstraintCode.
size() == 1) {
20160 switch (ConstraintCode[0]) {
20175 if (Constraint.
size() == 1) {
20176 switch (Constraint[0]) {
20179 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20181 if (isInt<12>(CVal))
20194 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20196 if (isUInt<5>(CVal))
20214 if (Subtarget.hasStdExtZtso()) {
20230 if (Subtarget.hasStdExtZtso()) {
20238 if (Subtarget.enableSeqCstTrailingFence() && isa<StoreInst>(Inst) &&
20255 if (Subtarget.hasForcedAtomics())
20260 if (Subtarget.hasStdExtZacas() &&
20261 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20267 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20280 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20282 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20284 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20286 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20288 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20290 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20292 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20294 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20303 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20305 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20307 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20309 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20311 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20313 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20315 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20317 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20343 unsigned XLen = Subtarget.
getXLen();
20367 unsigned ValWidth =
20372 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20375 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20387 if (Subtarget.hasForcedAtomics())
20391 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20400 unsigned XLen = Subtarget.
getXLen();
20402 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20407 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20413 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20420 EVT DataVT)
const {
20436 return Subtarget.hasStdExtZfhmin();
20438 return Subtarget.hasStdExtF();
20440 return Subtarget.hasStdExtD();
20472 "RVVBitsPerBlock changed, audit needed");
20481 if (!Subtarget.hasVendorXTHeadMemIdx())
20487 Base =
Op->getOperand(0);
20489 int64_t RHSC =
RHS->getSExtValue();
20495 bool isLegalIndexedOffset =
false;
20496 for (
unsigned i = 0; i < 4; i++)
20497 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20498 isLegalIndexedOffset =
true;
20502 if (!isLegalIndexedOffset)
20519 VT = LD->getMemoryVT();
20520 Ptr = LD->getBasePtr();
20521 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20522 VT = ST->getMemoryVT();
20523 Ptr = ST->getBasePtr();
20542 VT = LD->getMemoryVT();
20543 Ptr = LD->getBasePtr();
20544 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20545 VT = ST->getMemoryVT();
20546 Ptr = ST->getBasePtr();
20589 const Constant *PersonalityFn)
const {
20594 const Constant *PersonalityFn)
const {
20618 const bool HasExtMOrZmmul =
20619 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul();
20628 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
20630 const APInt &Imm = ConstNode->getAPIntValue();
20631 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
20632 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
20636 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
20637 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
20638 (Imm - 8).isPowerOf2()))
20643 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
20644 ConstNode->hasOneUse()) {
20645 APInt ImmS = Imm.ashr(Imm.countr_zero());
20646 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
20647 (1 - ImmS).isPowerOf2())
20671 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
20680 unsigned *
Fast)
const {
20683 *
Fast = Subtarget.enableUnalignedScalarMem();
20684 return Subtarget.enableUnalignedScalarMem();
20700 *
Fast = Subtarget.enableUnalignedVectorMem();
20701 return Subtarget.enableUnalignedVectorMem();
20710 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
20722 if (
Op.size() < MinVLenInBytes)
20732 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
20736 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
20738 if (
Op.isFixedDstAlign())
20739 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
20741 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
20749 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
20750 bool IsABIRegCopy =
CC.has_value();
20752 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20753 PartVT == MVT::f32) {
20771 if (PartVTBitSize % ValueVTBitSize == 0) {
20772 assert(PartVTBitSize >= ValueVTBitSize);
20779 if (ValueEltVT != PartEltVT) {
20780 if (PartVTBitSize > ValueVTBitSize) {
20782 assert(Count != 0 &&
"The number of element should not be zero.");
20783 EVT SameEltTypeVT =
20804 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
20805 bool IsABIRegCopy =
CC.has_value();
20806 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20807 PartVT == MVT::f32) {
20824 if (PartVTBitSize % ValueVTBitSize == 0) {
20825 assert(PartVTBitSize >= ValueVTBitSize);
20826 EVT SameEltTypeVT = ValueVT;
20833 if (ValueEltVT != PartEltVT) {
20835 assert(Count != 0 &&
"The number of element should not be zero.");
20852 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
20859 unsigned Opc =
N->getOpcode();
20883 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
20897 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
20902 if (FVTy->getNumElements() < 2)
20912 return Factor * LMUL <= 8;
20916 Align Alignment)
const {
20928 if (!Subtarget.enableUnalignedVectorMem() &&
20936 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
20937 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
20938 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
20939 Intrinsic::riscv_seg8_load};
20958 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
20968 {VTy, LI->getPointerOperandType(), XLenTy});
20970 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
20975 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
20977 Shuffles[i]->replaceAllUsesWith(SubVec);
20984 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
20985 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
20986 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
20987 Intrinsic::riscv_seg8_store};
21007 unsigned Factor)
const {
21009 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21012 ShuffleVTy->getNumElements() / Factor);
21014 SI->getPointerAddressSpace(),
21015 SI->getModule()->getDataLayout()))
21022 {VTy, SI->getPointerOperandType(), XLenTy});
21027 for (
unsigned i = 0; i < Factor; i++) {
21036 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21037 Ops.
append({SI->getPointerOperand(), VL});
21050 if (DI->
getIntrinsicID() != Intrinsic::experimental_vector_deinterleave2)
21053 unsigned Factor = 2;
21068 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21071 {ResVTy, LI->getPointerOperandType(), XLenTy});
21072 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21075 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21076 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21077 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21078 Intrinsic::riscv_vlseg8};
21100 if (II->
getIntrinsicID() != Intrinsic::experimental_vector_interleave2)
21103 unsigned Factor = 2;
21109 SI->getPointerAddressSpace(),
21110 SI->getModule()->getDataLayout()))
21117 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21120 {InVTy, SI->getPointerOperandType(), XLenTy});
21121 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21124 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21125 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21126 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21127 Intrinsic::riscv_vsseg8};
21135 SI->getPointerOperand(), VL});
21145 "Invalid call instruction for a KCFI check");
21147 MBBI->getOpcode()));
21150 Target.setIsRenamable(
false);
21158#define GET_REGISTER_MATCHER
21159#include "RISCVGenAsmMatcher.inc"
21165 if (Reg == RISCV::NoRegister)
21167 if (Reg == RISCV::NoRegister)
21179 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21181 if (NontemporalInfo ==
nullptr)
21189 int NontemporalLevel = 5;
21190 const MDNode *RISCVNontemporalInfo =
21191 I.getMetadata(
"riscv-nontemporal-domain");
21192 if (RISCVNontemporalInfo !=
nullptr)
21195 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21199 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21200 "RISC-V target doesn't support this non-temporal domain.");
21202 NontemporalLevel -= 2;
21204 if (NontemporalLevel & 0b1)
21206 if (NontemporalLevel & 0b10)
21219 return TargetFlags;
21229 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21232 return Subtarget.hasStdExtZbb() &&
21245 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21246 Op == Instruction::And ||
Op == Instruction::Or ||
21247 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21248 Op == Instruction::ShuffleVector ||
Op == Instruction::Load)
21256 !isa<ReturnInst>(&Inst))
21259 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21260 if (AI->getAllocatedType()->isScalableTy())
21268RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21276 if (!Subtarget.hasShortForwardBranchOpt())
21278 EVT VT =
N->getValueType(0);
21279 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21283 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21288bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21289 EVT VT,
const APInt &AndMask)
const {
21290 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21291 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21295unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21300template <
typename Arg>
21301void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21304 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21306 auto It = ArgList.
begin();
21307 MVT FirstArgRegType = It->VT;
21310 if (It == ArgList.
end() || It->Flags.isSplit())
21322 for (; It != ArgList.
end(); ++It)
21323 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21329 if (isHomogeneousScalableVectorType(ArgList)) {
21331 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21334 bool FirstVMaskAssigned =
false;
21335 for (
const auto &OutArg : ArgList) {
21336 MVT RegisterVT = OutArg.VT;
21346 RVVArgInfos.push_back({1, RegisterVT,
true});
21347 FirstVMaskAssigned =
true;
21351 RVVArgInfos.push_back({1, RegisterVT,
false});
21358void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21363 bool FirstVMaskAssigned =
false;
21364 for (
Type *Ty : TypeList) {
21374 RVVArgInfos.push_back(
21380 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21395 if (!FirstVMaskAssigned &&
21397 RVVArgInfos.push_back({1, RegisterVT,
true});
21398 FirstVMaskAssigned =
true;
21402 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21408void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21409 unsigned StartReg) {
21410 assert((StartReg % LMul) == 0 &&
21411 "Start register number should be multiple of lmul");
21430 for (
unsigned i = 0; i < NF; ++i)
21432 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
21434 AllocatedPhysRegs.push_back(
MCPhysReg());
21440void RVVArgDispatcher::compute() {
21442 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
21445 AllocatedPhysRegs.push_back(RISCV::V0);
21451 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
21452 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
21453 StartReg += RegsNeeded) {
21454 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
21455 if ((AssignedMap & Map) == 0) {
21456 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
21457 AssignedMap |=
Map;
21462 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
21465 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
21466 allocate(RVVArgInfos[i]);
21470 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
21471 return AllocatedPhysRegs[CurIdx++];
21476#define GET_RISCVVIntrinsicsTable_IMPL
21477#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static bool IsSelect(MachineInstr &MI)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Helper struct to store a base, index and offset that forms an address.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
bool hasScalarOperand() const
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)