34 #include "llvm/IR/IntrinsicsRISCV.h"
44 #define DEBUG_TYPE "riscv-lower"
46 STATISTIC(NumTailCalls,
"Number of tail calls");
60 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
61 "doesn't support the F instruction set extension (ignoring "
66 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
67 "doesn't support the D instruction set extension (ignoring "
114 auto addRegClassForRVV = [
this](
MVT VT) {
118 if (VT.getVectorMinNumElements() < MinElts)
121 unsigned Size = VT.getSizeInBits().getKnownMinValue();
124 RC = &RISCV::VRRegClass;
126 RC = &RISCV::VRM2RegClass;
128 RC = &RISCV::VRM4RegClass;
130 RC = &RISCV::VRM8RegClass;
137 for (
MVT VT : BoolVecVTs)
138 addRegClassForRVV(VT);
139 for (
MVT VT : IntVecVTs) {
140 if (VT.getVectorElementType() ==
MVT::i64 &&
143 addRegClassForRVV(VT);
147 for (
MVT VT : F16VecVTs)
148 addRegClassForRVV(VT);
151 for (
MVT VT : F32VecVTs)
152 addRegClassForRVV(VT);
155 for (
MVT VT : F64VecVTs)
156 addRegClassForRVV(VT);
159 auto addRegClassForFixedVectors = [
this](
MVT VT) {
166 if (useRVVForFixedLengthVectorVT(VT))
167 addRegClassForFixedVectors(VT);
170 if (useRVVForFixedLengthVectorVT(VT))
171 addRegClassForFixedVectors(VT);
213 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
318 for (
auto NT : FPLegalNodeTypes)
343 for (
auto NT : FPLegalNodeTypes)
349 for (
auto Op : FPOpToExpand)
359 for (
auto NT : FPLegalNodeTypes)
369 for (
auto Op : FPOpToExpand)
438 static const unsigned IntegerVPOps[] = {
439 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
440 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
441 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
442 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
443 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
444 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
445 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
446 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FPTOSI,
447 ISD::VP_FPTOUI, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
448 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE};
450 static const unsigned FloatingPointVPOps[] = {
451 ISD::VP_FADD, ISD::VP_FSUB,
452 ISD::VP_FMUL, ISD::VP_FDIV,
453 ISD::VP_FNEG, ISD::VP_FMA,
454 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
455 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX,
456 ISD::VP_MERGE, ISD::VP_SELECT,
457 ISD::VP_SITOFP, ISD::VP_UITOFP,
458 ISD::VP_SETCC, ISD::VP_FP_ROUND,
474 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
475 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
476 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
480 for (
MVT VT : BoolVecVTs) {
506 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
526 {ISD::VP_FPTOSI, ISD::VP_FPTOUI, ISD::VP_TRUNCATE, ISD::VP_SETCC}, VT,
531 for (
MVT VT : IntVecVTs) {
594 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
617 if (VT.getVectorElementType() !=
MVT::i64) {
642 const auto SetCommonVFPActions = [&](
MVT VT) {
691 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
708 const auto SetCommonVFPExtLoadTruncStoreActions =
710 for (
auto SmallVT : SmallerVTs) {
717 for (
MVT VT : F16VecVTs) {
720 SetCommonVFPActions(VT);
725 for (
MVT VT : F32VecVTs) {
728 SetCommonVFPActions(VT);
729 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
734 for (
MVT VT : F64VecVTs) {
737 SetCommonVFPActions(VT);
738 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
739 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
745 if (!useRVVForFixedLengthVectorVT(VT))
782 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
796 {ISD::VP_FPTOSI, ISD::VP_FPTOUI, ISD::VP_SETCC, ISD::VP_TRUNCATE},
818 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
866 if (!useRVVForFixedLengthVectorVT(VT))
891 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
904 for (
auto CC : VFPCCToExpand)
984 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
991 unsigned Intrinsic)
const {
992 auto &
DL =
I.getModule()->getDataLayout();
996 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
997 case Intrinsic::riscv_masked_atomicrmw_add_i32:
998 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
999 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1000 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1001 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1002 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1003 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1004 case Intrinsic::riscv_masked_cmpxchg_i32:
1007 Info.ptrVal =
I.getArgOperand(0);
1013 case Intrinsic::riscv_masked_strided_load:
1015 Info.ptrVal =
I.getArgOperand(1);
1017 Info.align =
Align(
DL.getTypeSizeInBits(
I.getType()->getScalarType()) / 8);
1021 case Intrinsic::riscv_masked_strided_store:
1023 Info.ptrVal =
I.getArgOperand(1);
1027 DL.getTypeSizeInBits(
I.getArgOperand(0)->getType()->getScalarType()) /
1032 case Intrinsic::riscv_seg2_load:
1033 case Intrinsic::riscv_seg3_load:
1034 case Intrinsic::riscv_seg4_load:
1035 case Intrinsic::riscv_seg5_load:
1036 case Intrinsic::riscv_seg6_load:
1037 case Intrinsic::riscv_seg7_load:
1038 case Intrinsic::riscv_seg8_load:
1040 Info.ptrVal =
I.getArgOperand(0);
1042 getValueType(
DL,
I.getType()->getStructElementType(0)->getScalarType());
1045 I.getType()->getStructElementType(0)->getScalarType()) /
1084 return isInt<12>(
Imm);
1088 return isInt<12>(
Imm);
1099 return (SrcBits == 64 && DestBits == 32);
1108 return (SrcBits == 64 && DestBits == 32);
1115 if (
auto *
LD = dyn_cast<LoadSDNode>(Val)) {
1116 EVT MemVT =
LD->getMemoryVT();
1143 EVT VT =
Y.getValueType();
1151 !isa<ConstantSDNode>(
Y);
1156 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1157 return C &&
C->getAPIntValue().ule(10);
1163 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1191 if (!
I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1195 switch (
I->getOpcode()) {
1197 case Instruction::Sub:
1199 case Instruction::And:
1200 case Instruction::Or:
1201 case Instruction::Xor:
1202 case Instruction::FAdd:
1203 case Instruction::FSub:
1204 case Instruction::FMul:
1205 case Instruction::FDiv:
1206 case Instruction::ICmp:
1207 case Instruction::FCmp:
1209 case Instruction::Shl:
1210 case Instruction::LShr:
1211 case Instruction::AShr:
1212 case Instruction::UDiv:
1213 case Instruction::SDiv:
1214 case Instruction::URem:
1215 case Instruction::SRem:
1216 return Operand == 1;
1218 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
1219 switch (II->getIntrinsicID()) {
1220 case Intrinsic::fma:
1221 case Intrinsic::vp_fma:
1222 return Operand == 0 || Operand == 1;
1226 case Intrinsic::vp_add:
1227 case Intrinsic::vp_mul:
1228 case Intrinsic::vp_and:
1229 case Intrinsic::vp_or:
1230 case Intrinsic::vp_xor:
1231 case Intrinsic::vp_fadd:
1232 case Intrinsic::vp_fmul:
1233 case Intrinsic::vp_shl:
1234 case Intrinsic::vp_lshr:
1235 case Intrinsic::vp_ashr:
1236 case Intrinsic::vp_udiv:
1237 case Intrinsic::vp_sdiv:
1238 case Intrinsic::vp_urem:
1239 case Intrinsic::vp_srem:
1240 return Operand == 1;
1243 case Intrinsic::vp_sub:
1244 case Intrinsic::vp_fsub:
1245 case Intrinsic::vp_fdiv:
1246 return Operand == 0 || Operand == 1;
1257 for (
auto OpIdx :
enumerate(
I->operands())) {
1258 if (!IsSinker(
I, OpIdx.index()))
1261 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
1263 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
1273 for (
Use &U :
Op->uses()) {
1279 Ops.push_back(&
Op->getOperandUse(0));
1280 Ops.push_back(&OpIdx.value());
1295 bool ForCodeSize)
const {
1303 return Imm.isZero();
1375 switch (KnownSize) {
1403 return RISCV::VRRegClassID;
1405 return RISCV::VRM2RegClassID;
1407 return RISCV::VRM4RegClassID;
1409 return RISCV::VRM8RegClassID;
1419 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1420 "Unexpected subreg numbering");
1421 return RISCV::sub_vrm1_0 +
Index;
1424 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1425 "Unexpected subreg numbering");
1426 return RISCV::sub_vrm2_0 +
Index;
1429 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1430 "Unexpected subreg numbering");
1431 return RISCV::sub_vrm4_0 +
Index;
1438 return RISCV::VRRegClassID;
1447 std::pair<unsigned, unsigned>
1449 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
1451 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1452 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1453 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1454 "Register classes not ordered");
1463 unsigned SubRegIdx = RISCV::NoSubRegister;
1464 for (
const unsigned RCID :
1465 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1466 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1475 return {SubRegIdx, InsertExtractIdx};
1480 bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
1509 "Unexpected opcode");
1511 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
1513 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1516 return Op.getOperand(II->
VLOperand + 1 + HasChain);
1519 static bool useRVVForFixedLengthVectorVT(
MVT VT,
1586 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
1595 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1596 "Expected legal fixed length vector!");
1599 unsigned MaxELen = Subtarget.
getELEN();
1639 "Expected to convert into a scalable vector!");
1641 "Expected a fixed length vector operand!");
1651 "Expected to convert into a fixed length vector!");
1653 "Expected a scalable vector operand!");
1679 static std::pair<SDValue, SDValue>
1692 static std::pair<SDValue, SDValue>
1708 EVT VT,
unsigned DefinedValues)
const {
1719 EVT DstVT =
Op.getValueType();
1720 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
1734 Opc,
DL, DstVT, Src,
1747 MVT VT =
Op.getSimpleValueType();
1770 Truncated = DAG.
getSelect(
DL, VT, NeedAdjust, Adjust, Truncated);
1779 Truncated = DAG.
getSelect(
DL, VT, NeedAdjust, Adjust, Truncated);
1798 return DAG.
getSelect(
DL, VT, Setcc, Truncated, Src);
1809 MVT VT =
Op.getSimpleValueType();
1824 Point5Pred.
next(
true);
1850 return DAG.
getSelect(
DL, VT, Setcc, Truncated, Src);
1870 unsigned NumElts =
Op.getNumOperands();
1872 if (!
Op.getValueType().isInteger())
1878 unsigned EltSizeInBits =
Op.getValueType().getScalarSizeInBits();
1879 for (
unsigned Idx = 0; Idx < NumElts; Idx++) {
1882 if (
Op.getOperand(Idx).isUndef())
1885 if (!isa<ConstantSDNode>(
Op.getOperand(Idx)))
1888 uint64_t Val =
Op.getConstantOperandVal(Idx) &
1889 maskTrailingOnes<uint64_t>(EltSizeInBits);
1894 unsigned IdxDiff = Idx - PrevElt->second;
1895 int64_t ValDiff =
SignExtend64(Val - PrevElt->first, EltSizeInBits);
1903 int64_t Remainder = ValDiff % IdxDiff;
1905 if (Remainder != ValDiff) {
1914 SeqStepNum = ValDiff;
1915 else if (ValDiff != SeqStepNum)
1919 SeqStepDenom = IdxDiff;
1920 else if (IdxDiff != *SeqStepDenom)
1925 if (!PrevElt || PrevElt->first != Val)
1926 PrevElt = std::make_pair(Val, Idx);
1930 if (!SeqStepNum || !SeqStepDenom)
1935 for (
unsigned Idx = 0; Idx < NumElts; Idx++) {
1936 if (
Op.getOperand(Idx).isUndef())
1938 uint64_t Val =
Op.getConstantOperandVal(Idx) &
1939 maskTrailingOnes<uint64_t>(EltSizeInBits);
1941 (int64_t)(Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
1942 int64_t Addend =
SignExtend64(Val - ExpectedVal, EltSizeInBits);
1945 else if (Addend != SeqAddend)
1949 assert(SeqAddend &&
"Must have an addend if we have a step");
1951 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1972 MVT ContainerVT = VT;
1992 MVT VT =
Op.getSimpleValueType();
2002 unsigned NumElts =
Op.getNumOperands();
2023 unsigned NumViaIntegerBits =
2034 MVT IntegerViaVecVT =
2039 unsigned BitPos = 0, IntegerEltIdx = 0;
2042 for (
unsigned I = 0;
I < NumElts;
I++, BitPos++) {
2045 if (
I != 0 &&
I % NumViaIntegerBits == 0) {
2046 if (NumViaIntegerBits <= 32)
2056 bool BitValue = !V.
isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2062 if (NumViaIntegerBits <= 32)
2068 if (NumElts < NumViaIntegerBits) {
2072 assert(IntegerViaVecVT ==
MVT::v1i8 &&
"Unexpected mask vector type");
2096 assert(Splat.getValueType() == XLenVT &&
2097 "Unexpected type for i1 splat value");
2125 int64_t StepNumerator = SimpleVID->StepNumerator;
2126 unsigned StepDenominator = SimpleVID->StepDenominator;
2127 int64_t Addend = SimpleVID->Addend;
2129 assert(StepNumerator != 0 &&
"Invalid step");
2130 bool Negate =
false;
2131 int64_t SplatStepVal = StepNumerator;
2133 if (StepNumerator != 1) {
2135 Negate = StepNumerator < 0;
2145 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
2146 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
2148 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
2155 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
2156 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
2159 VID = DAG.
getNode(StepOpcode,
DL, VT, VID, SplatStep);
2161 if (StepDenominator != 1) {
2166 if (Addend != 0 || Negate) {
2184 const auto *BV = cast<BuildVectorSDNode>(
Op);
2185 if (VT.
isInteger() && EltBitSize < 64 &&
2187 BV->getRepeatedSequence(
Sequence) &&
2188 (
Sequence.size() * EltBitSize) <= 64) {
2194 "Unexpected sequence type");
2196 unsigned EltIdx = 0;
2197 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2201 for (
const auto &SeqV :
Sequence) {
2202 if (!SeqV.isUndef())
2203 SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2204 << (EltIdx * EltBitSize));
2211 SplatValue = SignExtend64<32>(SplatValue);
2218 "Unexpected bitcast sequence");
2222 MVT ViaContainerVT =
2242 unsigned MostCommonCount = 0;
2244 unsigned NumUndefElts =
2252 unsigned NumScalarLoads = 0;
2258 ValueCounts.
insert(std::make_pair(V, 0));
2259 unsigned &Count = ValueCounts[V];
2261 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2262 NumScalarLoads += !CFP->isExactlyValue(+0.0);
2267 if (++Count >= MostCommonCount) {
2269 MostCommonCount = Count;
2273 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
2274 unsigned NumDefElts = NumElts - NumUndefElts;
2275 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2280 ((MostCommonCount > DominantValueCountThreshold) ||
2288 const SDValue &V = OpIdx.value();
2289 if (V.
isUndef() || !Processed.insert(V).second)
2291 if (ValueCounts[V] == 1) {
2300 return DAG.getConstant(V == V1, DL, XLenVT);
2319 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2320 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2321 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2324 if ((LoC >> 31) == HiC)
2329 auto *Const = dyn_cast<ConstantSDNode>(VL);
2330 if (LoC == HiC && Const && Const->isAllOnesValue()) {
2366 bool HasPassthru = Passthru && !Passthru.
isUndef();
2367 if (!HasPassthru && !Passthru)
2379 if (Scalar.getValueType().bitsLE(XLenVT)) {
2386 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
2391 (!Const ||
isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2397 "Unexpected scalar for splat lowering!");
2413 int Size =
Mask.size();
2416 int Srcs[] = {-1, -1};
2417 for (
int i = 0;
i != Size; ++
i) {
2426 int Src =
Mask[
i] / Size;
2429 if (Srcs[Pol] != Src)
2434 int Elt =
Mask[
i] % Size;
2440 if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2444 SwapSources = Srcs[0] > Srcs[1];
2462 int Size =
Mask.size();
2474 for (
int i = 0;
i != Size; ++
i) {
2480 int StartIdx =
i - (
M % Size);
2488 int CandidateRotation = StartIdx < 0 ? -StartIdx : Size - StartIdx;
2491 Rotation = CandidateRotation;
2492 else if (Rotation != CandidateRotation)
2497 int MaskSrc =
M < Size ? 0 : 1;
2502 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
2507 TargetSrc = MaskSrc;
2508 else if (TargetSrc != MaskSrc)
2515 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
2516 assert((LoSrc >= 0 || HiSrc >= 0) &&
2517 "Failed to find a rotated input vector!");
2528 MVT VT =
Op.getSimpleValueType();
2552 Offset %= OpElements;
2557 auto *Ld = cast<LoadSDNode>(V);
2567 SDValue Ops[] = {Ld->getChain(),
2585 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
2586 Ld->getPointerInfo().getWithOffset(Offset),
2587 Ld->getOriginalAlign(),
2591 Ld->getPointerInfo().getWithOffset(Offset), SVT,
2592 Ld->getOriginalAlign(),
2593 Ld->getMemOperand()->getFlags());
2604 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
2607 TrueMask, DAG.
getUNDEF(ContainerVT), VL);
2621 LoV = LoSrc == 0 ? V1 :
V2;
2625 HiV = HiSrc == 0 ? V1 :
V2;
2631 unsigned InvRotate = NumElts - Rotation;
2675 MVT WideIntContainerVT =
2707 V2, Multiplier, TrueMask, VL);
2727 int MaskIndex = MaskIdx.value();
2728 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
2731 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
2743 bool InvertMask = IsSelect == SwapOps;
2752 for (
int MaskIndex :
Mask) {
2753 bool SelectMaskVal = (MaskIndex < (
int)NumElts) ^ InvertMask;
2754 MaskVals.push_back(DAG.
getConstant(SelectMaskVal,
DL, XLenVT));
2756 bool IsLHSOrUndefIndex = MaskIndex < (
int)NumElts;
2757 GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2760 GatherIndicesRHS.push_back(
2761 IsLHSOrUndefIndex ? DAG.
getUNDEF(XLenVT)
2763 if (IsLHSOrUndefIndex && MaskIndex >= 0)
2764 ++LHSIndexCounts[MaskIndex];
2765 if (!IsLHSOrUndefIndex)
2766 ++RHSIndexCounts[MaskIndex - NumElts];
2772 std::swap(GatherIndicesLHS, GatherIndicesRHS);
2775 assert(MaskVals.size() == NumElts &&
"Unexpected select-like shuffle");
2801 MVT IndexContainerVT =
2815 if (LHSIndexCounts.
size() == 1) {
2816 int SplatIndex = LHSIndexCounts.
begin()->getFirst();
2817 Gather = DAG.
getNode(GatherVXOpc,
DL, ContainerVT, V1,
2825 Gather = DAG.
getNode(GatherVVOpc,
DL, ContainerVT, V1, LHSIndices,
2826 TrueMask, DAG.
getUNDEF(ContainerVT), VL);
2832 if (!
V2.isUndef()) {
2842 if (RHSIndexCounts.
size() == 1) {
2843 int SplatIndex = RHSIndexCounts.
begin()->getFirst();
2844 Gather = DAG.
getNode(GatherVXOpc,
DL, ContainerVT,
V2,
2851 Gather = DAG.
getNode(GatherVVOpc,
DL, ContainerVT,
V2, RHSIndices,
2852 SelectMask, Gather, VL);
2879 MVT VT =
Op.getSimpleValueType();
2893 "Expected legal float type!");
2908 unsigned ShiftAmt = FloatEltVT ==
MVT::f64 ? 52 : 23;
2914 unsigned ExponentBias = FloatEltVT ==
MVT::f64 ? 1023 : 127;
2923 unsigned Adjust = ExponentBias + (EltSize - 1);
2933 auto *
Load = cast<LoadSDNode>(
Op);
2934 assert(
Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
2937 Load->getMemoryVT(),
2938 *
Load->getMemOperand()))
2942 MVT VT =
Op.getSimpleValueType();
2944 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2945 "Unexpected unaligned RVV load type");
2949 "Expecting equally-sized RVV vector types to be legal");
2951 Load->getPointerInfo(),
Load->getOriginalAlign(),
2952 Load->getMemOperand()->getFlags());
2962 auto *
Store = cast<StoreSDNode>(
Op);
2964 "Expected vector store");
2967 Store->getMemoryVT(),
2968 *
Store->getMemOperand()))
2975 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2976 "Unexpected unaligned RVV store type");
2980 "Expecting equally-sized RVV vector types to be legal");
2981 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
2983 Store->getPointerInfo(),
Store->getOriginalAlign(),
2984 Store->getMemOperand()->getFlags());
2991 int64_t
Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
3015 switch (
Op.getOpcode()) {
3019 return lowerGlobalAddress(
Op, DAG);
3021 return lowerBlockAddress(
Op, DAG);
3023 return lowerConstantPool(
Op, DAG);
3025 return lowerJumpTable(
Op, DAG);
3027 return lowerGlobalTLSAddress(
Op, DAG);
3031 return lowerSELECT(
Op, DAG);
3033 return lowerBRCOND(
Op, DAG);
3035 return lowerVASTART(
Op, DAG);
3037 return lowerFRAMEADDR(
Op, DAG);
3039 return lowerRETURNADDR(
Op, DAG);
3041 return lowerShiftLeftParts(
Op, DAG);
3043 return lowerShiftRightParts(
Op, DAG,
true);
3045 return lowerShiftRightParts(
Op, DAG,
false);
3048 EVT VT =
Op.getValueType();
3074 "Unexpected types");
3108 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
3110 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
3112 return LowerINTRINSIC_VOID(
Op, DAG);
3115 MVT VT =
Op.getSimpleValueType();
3138 MVT VT =
Op.getSimpleValueType();
3139 assert(VT == Subtarget.
getXLenVT() &&
"Unexpected custom legalization");
3144 unsigned ShAmtWidth = Subtarget.
getXLen() - 1;
3159 return DAG.
getNode(Opc,
DL, VT, Op0, Op1, ShAmt);
3163 if (!
Op.getSimpleValueType().isVector())
3165 return lowerVectorTruncLike(
Op, DAG);
3168 if (
Op.getOperand(0).getValueType().isVector() &&
3169 Op.getOperand(0).getValueType().getVectorElementType() ==
MVT::i1)
3170 return lowerVectorMaskExt(
Op, DAG, 1);
3173 if (
Op.getOperand(0).getValueType().isVector() &&
3174 Op.getOperand(0).getValueType().getVectorElementType() ==
MVT::i1)
3175 return lowerVectorMaskExt(
Op, DAG, -1);
3178 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
3180 return lowerINSERT_VECTOR_ELT(
Op, DAG);
3182 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
3184 MVT VT =
Op.getSimpleValueType();
3220 Op.getOperand(1).getValueType() ==
MVT::i32) {
3232 if (!
Op.getValueType().isVector())
3234 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
3242 MVT VT =
Op.getSimpleValueType();
3248 MVT SrcVT = Src.getSimpleValueType();
3253 "Unexpected vector element types");
3257 if (EltSize > (2 * SrcEltSize)) {
3269 assert(SrcEltVT ==
MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
3278 if (SrcEltSize > (2 * EltSize)) {
3301 unsigned RVVOpc = 0;
3302 switch (
Op.getOpcode()) {
3319 MVT ContainerVT, SrcContainerVT;
3321 if (SrcEltSize > EltSize) {
3351 return lowerVECREDUCE(
Op, DAG);
3355 if (
Op.getOperand(0).getValueType().getVectorElementType() ==
MVT::i1)
3356 return lowerVectorMaskVecReduction(
Op, DAG,
false);
3357 return lowerVECREDUCE(
Op, DAG);
3362 return lowerFPVECREDUCE(
Op, DAG);
3363 case ISD::VP_REDUCE_ADD:
3364 case ISD::VP_REDUCE_UMAX:
3365 case ISD::VP_REDUCE_SMAX:
3366 case ISD::VP_REDUCE_UMIN:
3367 case ISD::VP_REDUCE_SMIN:
3368 case ISD::VP_REDUCE_FADD:
3369 case ISD::VP_REDUCE_SEQ_FADD:
3370 case ISD::VP_REDUCE_FMIN:
3371 case ISD::VP_REDUCE_FMAX:
3372 return lowerVPREDUCE(
Op, DAG);
3373 case ISD::VP_REDUCE_AND:
3374 case ISD::VP_REDUCE_OR:
3375 case ISD::VP_REDUCE_XOR:
3376 if (
Op.getOperand(1).getValueType().getVectorElementType() ==
MVT::i1)
3377 return lowerVectorMaskVecReduction(
Op, DAG,
true);
3378 return lowerVPREDUCE(
Op, DAG);
3380 return lowerINSERT_SUBVECTOR(
Op, DAG);
3382 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
3384 return lowerSTEP_VECTOR(
Op, DAG);
3386 return lowerVECTOR_REVERSE(
Op, DAG);
3388 return lowerVECTOR_SPLICE(
Op, DAG);
3392 if (
Op.getValueType().getVectorElementType() ==
MVT::i1)
3393 return lowerVectorMaskSplat(
Op, DAG);
3401 MVT VT =
Op.getSimpleValueType();
3402 unsigned NumOpElts =
3403 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3406 SDValue SubVec = OpIdx.value();
3416 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
3418 if (
Op.getValueType().isFixedLengthVector())
3419 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
3422 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
3424 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
3425 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
3429 return lowerMaskedLoad(
Op, DAG);
3432 return lowerMaskedStore(
Op, DAG);
3434 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
3465 if (
Op.getSimpleValueType().isFixedLengthVector())
3466 return lowerFixedLengthVectorShiftToRVV(
Op, DAG);
3469 "Unexpected custom legalisation");
3508 return lowerABS(
Op, DAG);
3513 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
3515 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
3517 case ISD::VP_GATHER:
3518 return lowerMaskedGather(
Op, DAG);
3520 case ISD::VP_SCATTER:
3521 return lowerMaskedScatter(
Op, DAG);
3523 return lowerGET_ROUNDING(
Op, DAG);
3525 return lowerSET_ROUNDING(
Op, DAG);
3527 return lowerEH_DWARF_CFA(
Op, DAG);
3528 case ISD::VP_SELECT:
3570 case ISD::VP_SIGN_EXTEND:
3571 case ISD::VP_ZERO_EXTEND:
3572 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() ==
MVT::i1)
3573 return lowerVPExtMaskOp(
Op, DAG);
3574 return lowerVPOp(
Op, DAG,
3575 Op.getOpcode() == ISD::VP_SIGN_EXTEND
3578 case ISD::VP_TRUNCATE:
3579 return lowerVectorTruncLike(
Op, DAG);
3580 case ISD::VP_FP_EXTEND:
3581 case ISD::VP_FP_ROUND:
3582 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
3583 case ISD::VP_FPTOSI:
3585 case ISD::VP_FPTOUI:
3587 case ISD::VP_SITOFP:
3589 case ISD::VP_UITOFP:
3592 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() ==
MVT::i1)
3593 return lowerVPSetCCMaskOp(
Op, DAG);
3612 N->getOffset(), Flags);
3620 template <
class NodeTy>
3622 bool IsLocal)
const {
3645 {DAG.getEntryNode(), Addr}, Ty,
MemOp);
3674 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
3678 return getAddr(
N, DAG, IsLocal);
3685 return getAddr(
N, DAG);
3692 return getAddr(
N, DAG);
3699 return getAddr(
N, DAG);
3704 bool UseGOT)
const {
3724 {DAG.getEntryNode(), Addr}, Ty,
MemOp);
3767 Args.push_back(Entry);
3784 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
3795 Addr = getStaticTLSAddr(
N, DAG,
false);
3798 Addr = getStaticTLSAddr(
N, DAG,
true);
3802 Addr = getDynamicTLSAddr(
N, DAG);
3814 MVT VT =
Op.getSimpleValueType();
3836 const auto *CC = cast<CondCodeSDNode>(CondV.
getOperand(2));
3846 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3848 const APInt &
TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3849 const APInt &
FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3869 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3889 LHS,
RHS, TargetCC,
Op.getOperand(2));
3907 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3919 int XLenInBytes = Subtarget.
getXLen() / 8;
3921 EVT VT =
Op.getValueType();
3924 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3926 int Offset = -(XLenInBytes * 2);
3942 int XLenInBytes = Subtarget.
getXLen() / 8;
3947 EVT VT =
Op.getValueType();
3949 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3951 int Off = -XLenInBytes;
3952 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
3971 EVT VT =
Lo.getValueType();
4010 EVT VT =
Lo.getValueType();
4061 MVT VT =
Op.getSimpleValueType();
4074 "Unexpected type for i1 splat value");
4090 MVT VecVT =
Op.getSimpleValueType();
4092 "Unexpected SPLAT_VECTOR_PARTS lowering");
4094 assert(
Op.getNumOperands() == 2 &&
"Unexpected number of operands!");
4102 std::tie(
Mask, VL) =
4110 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4111 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4112 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4115 if ((LoC >> 31) == HiC)
4121 if (
Hi.getOpcode() ==
ISD::SRA &&
Hi.getOperand(0) == Lo &&
4122 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4123 Hi.getConstantOperandVal(1) == 31)
4138 int64_t ExtTrueVal)
const {
4140 MVT VecVT =
Op.getSimpleValueType();
4143 assert(Src.getValueType().isVector() &&
4144 Src.getValueType().getVectorElementType() ==
MVT::i1);
4166 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
4168 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
4170 SplatTrueVal, SplatZero, VL);
4175 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4177 MVT ExtVT =
Op.getSimpleValueType();
4181 MVT VT =
Op.getOperand(0).getSimpleValueType();