RISCVISelLowering.cpp

Go to the documentation of this file.

//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation  -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the interfaces that RISC-V uses to lower LLVM code into a
// selection DAG.
//
//===----------------------------------------------------------------------===//
 
#include "RISCVISelLowering.h"
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCV.h"
#include "RISCVConstantPoolValue.h"
#include "RISCVMachineFunctionInfo.h"
#include "RISCVRegisterInfo.h"
#include "RISCVSelectionDAGInfo.h"
#include "RISCVSubtarget.h"
#include "RISCVVectorUtils.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
 
using namespace llvm;
 
#define DEBUG_TYPE "riscv-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
 
static cl::opt<unsigned> ExtensionMaxWebSize(
    DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
    cl::desc("Give the maximum size (in number of nodes) of the web of "
             "instructions that we will consider for VW expansion"),
    cl::init(18));
 
static cl::opt<bool>
    AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
                     cl::desc("Allow the formation of VW_W operations (e.g., "
                              "VWADD_W) with splat constants"),
                     cl::init(false));
 
static cl::opt<unsigned> NumRepeatedDivisors(
    DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
    cl::desc("Set the minimum number of repetitions of a divisor to allow "
             "transformation to multiplications by the reciprocal"),
    cl::init(2));
 
static cl::opt<int>
    FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
              cl::desc("Give the maximum number of instructions that we will "
                       "use for creating a floating-point immediate value"),
              cl::init(3));
 
static cl::opt<bool>
    ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
                      cl::desc("Swap add and addi in cases where the add may "
                               "be combined with a shift"),
                      cl::init(true));
 
static cl::opt<int> BrMergingBaseCostThresh(
    "riscv-br-merging-base-cost", cl::init(2),
    cl::desc(
        "Sets the cost threshold for when multiple conditionals will be merged "
        "into one branch versus be split in multiple branches. Merging "
        "conditionals saves branches at the cost of additional instructions. "
        "This value sets the instruction cost limit, below which conditionals "
        "will be merged, and above which conditionals will be split. Set to -1 "
        "to never merge branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingLikelyBias(
    "riscv-br-merging-likely-bias", cl::init(0),
    cl::desc(
        "Increases 'riscv-br-merging-base-cost' in cases that it is "
        "likely that all conditionals will be executed. For example for "
        "merging the conditionals (a == b && c > d), if its known that "
        "a == b is likely, then it is likely that if the conditionals are "
        "split both sides will be executed, so it may be desirable to "
        "increase the instruction cost threshold. Set to -1 to never merge "
        "likely branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingUnlikelyBias(
    "riscv-br-merging-unlikely-bias", cl::init(-1),
    cl::desc(
        "Decreases 'riscv-br-merging-base-cost' in cases that it is unlikely "
        "that all conditionals will be executed. For example for merging "
        "the conditionals (a == b && c > d), if its known that a == b is "
        "unlikely, then it is unlikely that if the conditionals are split "
        "both sides will be executed, so it may be desirable to decrease "
        "the instruction cost threshold. Set to -1 to never merge unlikely "
        "branches."),
    cl::Hidden);
 
// TODO: Support more ops
static const unsigned ZvfbfaOps[] = {
    ISD::FNEG,        ISD::FABS,        ISD::FCOPYSIGN,   ISD::FADD,
    ISD::FSUB,        ISD::FMUL,        ISD::FMINNUM,     ISD::FMAXNUM,
    ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUM,    ISD::FMAXIMUM,
    ISD::FMA,         ISD::IS_FPCLASS,  ISD::STRICT_FADD, ISD::STRICT_FSUB,
    ISD::STRICT_FMUL, ISD::STRICT_FMA,  ISD::SETCC};
 
RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                                         const RISCVSubtarget &STI)
    : TargetLowering(TM, STI), Subtarget(STI) {
 
  RISCVABI::ABI ABI = Subtarget.getTargetABI();
  // Note: Hard-float ABIs that don't match the F/D extensions are already
  // rejected/ by RISCVABI::computeTargetABI() during subtarget construction.
  assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
 
  switch (ABI) {
  default:
    reportFatalUsageError("Don't know how to lower this ABI");
  case RISCVABI::ABI_ILP32:
  case RISCVABI::ABI_ILP32E:
  case RISCVABI::ABI_LP64E:
  case RISCVABI::ABI_ILP32F:
  case RISCVABI::ABI_ILP32D:
  case RISCVABI::ABI_LP64:
  case RISCVABI::ABI_LP64F:
  case RISCVABI::ABI_LP64D:
    break;
  }
 
  MVT XLenVT = Subtarget.getXLenVT();
 
  // Set up the register classes.
  addRegisterClass(XLenVT, &RISCV::GPRRegClass);
 
  if (Subtarget.hasStdExtZfhmin())
    addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
  if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
    addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);
  if (Subtarget.hasStdExtF())
    addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
  if (Subtarget.hasStdExtD())
    addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
  if (Subtarget.hasStdExtZhinxmin())
    addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);
  if (Subtarget.hasStdExtZfinx())
    addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
  if (Subtarget.hasStdExtZdinx()) {
    if (Subtarget.is64Bit())
      addRegisterClass(MVT::f64, &RISCV::GPRRegClass);
    else
      addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);
  }
 
  static const MVT::SimpleValueType BoolVecVTs[] = {
      MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
      MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
  static const MVT::SimpleValueType IntVecVTs[] = {
      MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
      MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
      MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
      MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
      MVT::nxv4i64, MVT::nxv8i64};
  static const MVT::SimpleValueType F16VecVTs[] = {
      MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
      MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
  static const MVT::SimpleValueType BF16VecVTs[] = {
      MVT::nxv1bf16, MVT::nxv2bf16,  MVT::nxv4bf16,
      MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
  static const MVT::SimpleValueType F32VecVTs[] = {
      MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
  static const MVT::SimpleValueType F64VecVTs[] = {
      MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
  static const MVT::SimpleValueType VecTupleVTs[] = {
      MVT::riscv_nxv1i8x2,  MVT::riscv_nxv1i8x3,  MVT::riscv_nxv1i8x4,
      MVT::riscv_nxv1i8x5,  MVT::riscv_nxv1i8x6,  MVT::riscv_nxv1i8x7,
      MVT::riscv_nxv1i8x8,  MVT::riscv_nxv2i8x2,  MVT::riscv_nxv2i8x3,
      MVT::riscv_nxv2i8x4,  MVT::riscv_nxv2i8x5,  MVT::riscv_nxv2i8x6,
      MVT::riscv_nxv2i8x7,  MVT::riscv_nxv2i8x8,  MVT::riscv_nxv4i8x2,
      MVT::riscv_nxv4i8x3,  MVT::riscv_nxv4i8x4,  MVT::riscv_nxv4i8x5,
      MVT::riscv_nxv4i8x6,  MVT::riscv_nxv4i8x7,  MVT::riscv_nxv4i8x8,
      MVT::riscv_nxv8i8x2,  MVT::riscv_nxv8i8x3,  MVT::riscv_nxv8i8x4,
      MVT::riscv_nxv8i8x5,  MVT::riscv_nxv8i8x6,  MVT::riscv_nxv8i8x7,
      MVT::riscv_nxv8i8x8,  MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
      MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
 
  if (Subtarget.hasVInstructions()) {
    auto addRegClassForRVV = [this](MVT VT) {
      // Disable the smallest fractional LMUL types if ELEN is less than
      // RVVBitsPerBlock.
      unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
      if (VT.getVectorMinNumElements() < MinElts)
        return;
 
      unsigned Size = VT.getSizeInBits().getKnownMinValue();
      const TargetRegisterClass *RC;
      if (Size <= RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRRegClass;
      else if (Size == 2 * RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRM2RegClass;
      else if (Size == 4 * RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRM4RegClass;
      else if (Size == 8 * RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRM8RegClass;
      else
        llvm_unreachable("Unexpected size");
 
      addRegisterClass(VT, RC);
    };
 
    for (MVT VT : BoolVecVTs)
      addRegClassForRVV(VT);
    for (MVT VT : IntVecVTs) {
      if (VT.getVectorElementType() == MVT::i64 &&
          !Subtarget.hasVInstructionsI64())
        continue;
      addRegClassForRVV(VT);
    }
 
    if (Subtarget.hasVInstructionsF16Minimal() ||
        Subtarget.hasVendorXAndesVPackFPH())
      for (MVT VT : F16VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.hasVInstructionsBF16Minimal() ||
        Subtarget.hasVendorXAndesVBFHCvt())
      for (MVT VT : BF16VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.hasVInstructionsF32())
      for (MVT VT : F32VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.hasVInstructionsF64())
      for (MVT VT : F64VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.useRVVForFixedLengthVectors()) {
      auto addRegClassForFixedVectors = [this](MVT VT) {
        MVT ContainerVT = getContainerForFixedLengthVector(VT);
        unsigned RCID = getRegClassIDForVecVT(ContainerVT);
        const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
        addRegisterClass(VT, TRI.getRegClass(RCID));
      };
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
        if (useRVVForFixedLengthVectorVT(VT))
          addRegClassForFixedVectors(VT);
 
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
        if (useRVVForFixedLengthVectorVT(VT))
          addRegClassForFixedVectors(VT);
    }
 
    addRegisterClass(MVT::riscv_nxv1i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv16i8x2, &RISCV::VRN2M2RegClass);
    addRegisterClass(MVT::riscv_nxv16i8x3, &RISCV::VRN3M2RegClass);
    addRegisterClass(MVT::riscv_nxv16i8x4, &RISCV::VRN4M2RegClass);
    addRegisterClass(MVT::riscv_nxv32i8x2, &RISCV::VRN2M4RegClass);
  }
 
  // fixed vector is stored in GPRs for P extension packed operations
  if (Subtarget.hasStdExtP()) {
    if (Subtarget.is64Bit()) {
      addRegisterClass(MVT::v2i32, &RISCV::GPRRegClass);
      addRegisterClass(MVT::v4i16, &RISCV::GPRRegClass);
      addRegisterClass(MVT::v8i8, &RISCV::GPRRegClass);
    } else {
      addRegisterClass(MVT::v2i16, &RISCV::GPRRegClass);
      addRegisterClass(MVT::v4i8, &RISCV::GPRRegClass);
 
      addRegisterClass(MVT::v2i32, &RISCV::GPRPairRegClass);
      addRegisterClass(MVT::v4i16, &RISCV::GPRPairRegClass);
      addRegisterClass(MVT::v8i8, &RISCV::GPRPairRegClass);
    }
  }
 
  // Compute derived properties from the register classes.
  computeRegisterProperties(STI.getRegisterInfo());
 
  setStackPointerRegisterToSaveRestore(RISCV::X2);
 
  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
                   MVT::i1, Promote);
  // DAGCombiner can call isLoadExtLegal for types that aren't legal.
  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
                   MVT::i1, Promote);
 
  // TODO: add all necessary setOperationAction calls.
  setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Custom);
 
  setOperationAction(ISD::BR_JT, MVT::Other, Expand);
  setOperationAction(ISD::BR_CC, XLenVT, Expand);
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
 
  setCondCodeAction(ISD::SETGT, XLenVT, Custom);
  setCondCodeAction(ISD::SETGE, XLenVT, Expand);
  setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
  setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
  if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
    setCondCodeAction(ISD::SETULE, XLenVT, Expand);
    setCondCodeAction(ISD::SETLE, XLenVT, Expand);
  }
 
  setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
 
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
  setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
 
  if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
      !Subtarget.hasVendorXAndesPerf())
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
  setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
 
  if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
      !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
      !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
    setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
 
  if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
    setOperationAction(ISD::LOAD, MVT::i64, Custom);
    setOperationAction(ISD::STORE, MVT::i64, Custom);
  }
 
  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
 
    setOperationAction(ISD::LOAD, MVT::i32, Custom);
    setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
                       MVT::i32, Custom);
    setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Custom);
    setOperationAction({ISD::SADDO, ISD::SSUBO}, MVT::i32, Custom);
  } else if (Subtarget.hasStdExtP()) {
    // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
    setOperationAction({ISD::ADD, ISD::SUB}, MVT::i64, Custom);
    setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
  }
  if (!Subtarget.hasStdExtZmmul()) {
    setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
  } else if (Subtarget.is64Bit()) {
    setOperationAction(ISD::MUL, MVT::i128, Custom);
    setOperationAction(ISD::MUL, MVT::i32, Custom);
  } else {
    setOperationAction(ISD::MUL, MVT::i64, Custom);
  }
 
  if (!Subtarget.hasStdExtM()) {
    setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, XLenVT,
                       Expand);
  } else if (Subtarget.is64Bit()) {
    setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
                       {MVT::i8, MVT::i16, MVT::i32}, Custom);
  }
 
  setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, XLenVT, Expand);
 
  // On RV32, the P extension has a WMUL(U) instruction we can use for
  // (S/U)MUL_LOHI.
  // FIXME: Does P imply Zmmul?
  if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
      Subtarget.is64Bit())
    setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, Expand);
 
  setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
                     Custom);
 
  if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
    if (Subtarget.is64Bit())
      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
  } else if (Subtarget.hasVendorXTHeadBb()) {
    if (Subtarget.is64Bit())
      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
    setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);
  } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
    setOperationAction(ISD::ROTL, XLenVT, Expand);
  } else {
    setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
  }
 
  if (Subtarget.hasStdExtP())
    setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Legal);
 
  setOperationAction(ISD::BSWAP, XLenVT,
                     Subtarget.hasREV8Like() ? Legal : Expand);
 
  if (Subtarget.hasREVLike()) {
    setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
  } else {
    // Zbkb can use rev8+brev8 to implement bitreverse.
    setOperationAction(ISD::BITREVERSE, XLenVT,
                       Subtarget.hasStdExtZbkb() ? Custom : Expand);
    if (Subtarget.hasStdExtZbkb())
      setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
  }
 
  if (Subtarget.hasStdExtZbb() ||
      (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
    setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
                       Legal);
  }
 
  if (Subtarget.hasCTZLike()) {
    if (Subtarget.is64Bit())
      setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, MVT::i32, Custom);
  } else {
    setOperationAction(ISD::CTTZ, XLenVT, Expand);
  }
 
  if (!Subtarget.hasCPOPLike()) {
    // TODO: These should be set to LibCall, but this currently breaks
    //   the Linux kernel build. See #101786. Lacks i128 tests, too.
    if (Subtarget.is64Bit())
      setOperationAction(ISD::CTPOP, MVT::i128, Expand);
    else
      setOperationAction(ISD::CTPOP, MVT::i32, Expand);
    setOperationAction(ISD::CTPOP, MVT::i64, Expand);
  }
 
  if (Subtarget.hasCLZLike()) {
    // We need the custom lowering to make sure that the resulting sequence
    // for the 32bit case is efficient on 64bit targets.
    // Use default promotion for i32 without Zbb.
    if (Subtarget.is64Bit() &&
        (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
      setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, MVT::i32, Custom);
  } else {
    if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
      setOperationAction(ISD::CTLZ_ZERO_POISON, XLenVT, Legal);
    setOperationAction(ISD::CTLZ, XLenVT, Expand);
  }
 
  if (Subtarget.hasStdExtP()) {
    setOperationAction(ISD::CTLS, XLenVT, Legal);
    if (Subtarget.is64Bit())
      setOperationAction(ISD::CTLS, MVT::i32, Custom);
  }
 
  if (Subtarget.hasStdExtP() ||
      (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
    setOperationAction(ISD::ABS, XLenVT, Legal);
    if (Subtarget.is64Bit())
      setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
  } else if (Subtarget.hasShortForwardBranchIALU()) {
    // We can use PseudoCCSUB to implement ABS.
    setOperationAction(ISD::ABS, XLenVT, Legal);
  } else if (Subtarget.is64Bit()) {
    setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
  }
 
  if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
    setOperationAction(ISD::SELECT, XLenVT, Custom);
 
  if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
      !Subtarget.is64Bit()) {
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i32, Legal);
  } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i32, Custom);
  } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i32, Custom);
  }
 
  if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
      !Subtarget.is64Bit()) {
    // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
    // pssha.w/psshl.w and extracting.
    setOperationAction(ISD::SSHLSAT, MVT::i32, Legal);
    setOperationAction(ISD::USHLSAT, MVT::i32, Legal);
  }
 
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
    // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
    // paadd.w, paaddu.w and extracting.
    setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, MVT::i32, Legal);
  }
 
  if (Subtarget.hasStdExtZbkc()) {
    setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Legal);
    if (Subtarget.hasStdExtZbc())
      setOperationAction(ISD::CLMULR, XLenVT, Legal);
  } else if (Subtarget.hasStdExtZvbc() && Subtarget.is64Bit()) {
    // FIXME: Support i32 on RV32 by zexting from XLEN to i64 and extracting
    // half of the result (low for CLMUL, high for CLMULH).
    // TODO: Zvbc32e allows us to do a lot more here.
    setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Custom);
  }
 
  static const unsigned FPLegalNodeTypes[] = {
      ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUMNUM,
      ISD::FMAXIMUMNUM,   ISD::LRINT,          ISD::LLRINT,
      ISD::LROUND,        ISD::LLROUND,        ISD::STRICT_LRINT,
      ISD::STRICT_LLRINT, ISD::STRICT_LROUND,  ISD::STRICT_LLROUND,
      ISD::STRICT_FMA,    ISD::STRICT_FADD,    ISD::STRICT_FSUB,
      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,    ISD::STRICT_FSQRT,
      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
 
  static const ISD::CondCode FPCCToExpand[] = {
      ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
      ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
      ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
 
  static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
                                          ISD::FPOW};
  static const unsigned FPOpToLibCall[] = {ISD::FREM};
 
  static const unsigned FPRndMode[] = {
      ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
      ISD::FROUNDEVEN};
 
  static const unsigned ZfhminZfbfminPromoteOps[] = {
      ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUM,
      ISD::FMAXIMUM,      ISD::FMAXIMUMNUM,    ISD::FMINIMUMNUM,
      ISD::FADD,          ISD::FSUB,           ISD::FMUL,
      ISD::FMA,           ISD::FDIV,           ISD::FSQRT,
      ISD::STRICT_FMA,    ISD::STRICT_FADD,    ISD::STRICT_FSUB,
      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,    ISD::STRICT_FSQRT,
      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
      ISD::FCEIL,         ISD::FFLOOR,         ISD::FTRUNC,
      ISD::FRINT,         ISD::FROUND,         ISD::FROUNDEVEN,
      ISD::FCANONICALIZE};
 
  if (Subtarget.hasStdExtP()) {
    static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
    static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
    ArrayRef<MVT> VTs;
    if (Subtarget.is64Bit()) {
      VTs = P64VecVTs;
      // There's no instruction for vector shamt in P extension so we unroll to
      // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
      // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
      // widen for those operations that will be unrolled.
      setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA},
                         {MVT::v2i16, MVT::v4i8}, Custom);
      setOperationAction(ISD::INTRINSIC_WO_CHAIN, {MVT::v2i16, MVT::v4i8},
                         Custom);
      // Operand legalization queries the action using the illegal subvector.
      setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
                         Custom);
    } else {
      VTs = P32VecVTs;
    }
    // By default everything must be expanded.
    for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
      setOperationAction(Op, VTs, Expand);
 
    for (MVT VT : VTs) {
      for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
        setTruncStoreAction(VT, OtherVT, Expand);
        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                         OtherVT, Expand);
      }
    }
 
    setOperationAction({ISD::LOAD, ISD::STORE}, VTs, Legal);
    setOperationAction({ISD::ADD, ISD::SUB}, VTs, Legal);
    setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VTs, Legal);
    setOperationAction(ISD::UADDSAT, VTs, Legal);
    setOperationAction(ISD::SADDSAT, VTs, Legal);
    setOperationAction(ISD::USUBSAT, VTs, Legal);
    setOperationAction(ISD::SSUBSAT, VTs, Legal);
    setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Legal);
    setOperationAction(ISD::BITREVERSE, VTs, Legal);
    setOperationAction(ISD::VECTOR_SHUFFLE, VTs, Custom);
    setOperationAction(ISD::VECTOR_REVERSE, VTs, Legal);
    for (MVT VT : VTs) {
      if (VT != MVT::v2i32)
        setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Legal);
      if (VT.getVectorElementType() != MVT::i8) {
        setOperationAction(ISD::SSHLSAT, VT, Custom);
        setOperationAction(ISD::BSWAP, VT, Legal);
      }
    }
    setOperationAction({ISD::UNDEF, ISD::POISON}, VTs, Legal);
    setOperationAction(ISD::SPLAT_VECTOR, VTs, Legal);
    setOperationAction(ISD::BUILD_VECTOR, VTs, Legal);
    setOperationAction(ISD::SCALAR_TO_VECTOR, VTs, Legal);
    setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Custom);
    setOperationAction(ISD::BITCAST, VTs, Custom);
    setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
                       Custom);
    setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
                       Legal);
    setOperationAction(ISD::SELECT, VTs, Custom);
    setOperationAction(ISD::VSELECT, VTs, Legal);
    setOperationAction(ISD::SETCC, VTs, Legal);
    setCondCodeAction(
        {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
        Expand);
    setCondCodeAction({ISD::SETNE, ISD::SETGT}, VTs, Custom);
 
    if (!Subtarget.is64Bit())
      setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v4i8}, Custom);
 
    // P extension vector comparisons produce all 1s for true, all 0s for false
    setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
    if (!Subtarget.is64Bit()) {
      // By default everything must be expanded.
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
        setOperationAction(Op, P64VecVTs, Expand);
 
      for (MVT VT : P64VecVTs) {
        for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
          setTruncStoreAction(VT, OtherVT, Expand);
          setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                           OtherVT, Expand);
        }
      }
 
      setOperationAction({ISD::UNDEF, ISD::POISON}, P64VecVTs, Legal);
      setOperationAction({ISD::LOAD, ISD::STORE}, P64VecVTs, Custom);
      setOperationAction(ISD::BITCAST, P64VecVTs, Custom);
      setOperationAction({ISD::ADD, ISD::SUB}, P64VecVTs, Legal);
      setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, {MVT::v4i16, MVT::v8i8},
                         Custom);
      setOperationAction(
          {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, P64VecVTs,
          Legal);
      setOperationAction(ISD::INTRINSIC_WO_CHAIN, P64VecVTs, Legal);
      setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
      setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, P64VecVTs, Legal);
      setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
                         P64VecVTs, Legal);
      setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU},
                         {MVT::v4i16, MVT::v8i8}, Legal);
      setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, P64VecVTs, Custom);
      setOperationAction(ISD::SSHLSAT, {MVT::v2i32, MVT::v4i16}, Custom);
      setOperationAction(ISD::BSWAP, MVT::v4i16, Legal);
      setOperationAction(ISD::BITREVERSE, {MVT::v4i16, MVT::v8i8}, Legal);
      setOperationAction(ISD::VECTOR_SHUFFLE, P64VecVTs, Custom);
      setOperationAction(ISD::VECTOR_REVERSE, P64VecVTs, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, P64VecVTs, Legal);
      setOperationAction(ISD::BUILD_VECTOR, P64VecVTs, Legal);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Legal);
      setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
                         {MVT::v4i16, MVT::v8i8}, Custom);
      setOperationAction(ISD::CONCAT_VECTORS, {MVT::v4i16, MVT::v8i8}, Legal);
      setOperationAction(ISD::EXTRACT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
                         Legal);
      setOperationAction({ISD::SELECT, ISD::VSELECT}, {MVT::v4i16, MVT::v8i8},
                         Custom);
      setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU},
                         {MVT::v4i16, MVT::v8i8}, Custom);
      setOperationAction(ISD::MUL, P32VecVTs, Custom);
      setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v4i8, Custom);
      setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v2i16, Legal);
      setOperationAction({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
                         {MVT::v4i16, MVT::v2i32}, Legal);
      setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Legal);
      setOperationAction(ISD::SETCC, P64VecVTs, Legal);
      setCondCodeAction(
          {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
          P64VecVTs, Expand);
      setCondCodeAction({ISD::SETNE, ISD::SETGT}, P64VecVTs, Custom);
      // Operation legalization queries the action using the result type.
      setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v4i16, MVT::v8i8},
                         Custom);
    } else {
      setOperationAction(ISD::MUL, P64VecVTs, Legal);
      setOperationAction({ISD::MULHS, ISD::MULHU}, {MVT::v2i32, MVT::v4i16},
                         Legal);
      setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v8i8, Custom);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG,
                         {MVT::v4i16, MVT::v2i32}, Legal);
      setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, {MVT::v4i16, MVT::v2i32},
                         Custom);
      setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Custom);
    }
    // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
    setOperationAction(
        ISD::SIGN_EXTEND_INREG,
        {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}, Legal);
  }
 
  if (Subtarget.hasStdExtZfbfmin()) {
    setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::bf16, Expand);
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
    setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
    setOperationAction(ISD::SELECT, MVT::bf16, Custom);
    setOperationAction(ISD::BR_CC, MVT::bf16, Expand);
    setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);
    setOperationAction(ISD::FREM, MVT::bf16, Promote);
    setOperationAction(ISD::FABS, MVT::bf16, Custom);
    setOperationAction(ISD::FNEG, MVT::bf16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
  }
 
  if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
    if (Subtarget.hasStdExtZfhmin())
      setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f16, Expand);
    if (Subtarget.hasStdExtZfhOrZhinx()) {
      setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
      setOperationAction(FPRndMode, MVT::f16,
                         Subtarget.hasStdExtZfa() ? Legal : Custom);
      setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
                         Subtarget.hasStdExtZfa() ? Legal : Custom);
      if (Subtarget.hasStdExtZfa())
        setOperationAction(ISD::ConstantFP, MVT::f16, Custom);
    } else {
      setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Promote);
      for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
                      ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
                      ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
        setOperationAction(Op, MVT::f16, Custom);
      setOperationAction(ISD::FABS, MVT::f16, Custom);
      setOperationAction(ISD::FNEG, MVT::f16, Custom);
      setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
      setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
    }
 
    if (!Subtarget.hasStdExtD()) {
      // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
      // instruction runs into double rounding issues, so this is wrong.
      // Normally we'd use an f64 fma, but without the D extension the f64 type
      // is not legal. This should probably be a libcall.
      AddPromotedToType(ISD::FMA, MVT::f16, MVT::f32);
      AddPromotedToType(ISD::STRICT_FMA, MVT::f16, MVT::f32);
    }
 
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
 
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
    setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
    setOperationAction(ISD::SELECT, MVT::f16, Custom);
    setOperationAction(ISD::BR_CC, MVT::f16, Expand);
 
    setOperationAction(
        ISD::FNEARBYINT, MVT::f16,
        Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
    setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
                        ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
                        ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
                        ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
                       MVT::f16, Promote);
 
    // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
    // complete support for all operations in LegalizeDAG.
    setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
                        ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
                        ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
                        ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
                       MVT::f16, Promote);
 
    // We need to custom promote this.
    if (Subtarget.is64Bit())
      setOperationAction(ISD::FPOWI, MVT::i32, Custom);
  }
 
  if (Subtarget.hasStdExtFOrZfinx()) {
    if (Subtarget.hasStdExtF())
      setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f32, Expand);
    setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
    setOperationAction(FPRndMode, MVT::f32,
                       Subtarget.hasStdExtZfa() ? Legal : Custom);
    setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
    setOperationAction(ISD::SELECT, MVT::f32, Custom);
    setOperationAction(ISD::BR_CC, MVT::f32, Expand);
    setOperationAction(FPOpToExpand, MVT::f32, Expand);
    setOperationAction(FPOpToLibCall, MVT::f32, LibCall);
    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
    setTruncStoreAction(MVT::f32, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
    setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
    setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
    setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
    setOperationAction(ISD::FP_TO_BF16, MVT::f32,
                       Subtarget.isSoftFPABI() ? LibCall : Custom);
    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
 
    if (Subtarget.hasStdExtZfa()) {
      setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
      setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
    } else {
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);
    }
  }
 
  if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
 
  if (Subtarget.hasStdExtDOrZdinx()) {
    if (Subtarget.hasStdExtD())
      setOperationAction({ISD::UNDEF, ISD::POISON}, MVT::f64, Expand);
    setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
 
    if (!Subtarget.is64Bit())
      setOperationAction(ISD::BITCAST, MVT::i64, Custom);
 
    if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
        !Subtarget.is64Bit()) {
      setOperationAction(ISD::LOAD, MVT::f64, Custom);
      setOperationAction(ISD::STORE, MVT::f64, Custom);
    }
 
    if (Subtarget.hasStdExtZfa()) {
      setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
      setOperationAction(FPRndMode, MVT::f64, Legal);
      setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Legal);
    } else {
      if (Subtarget.is64Bit())
        setOperationAction(FPRndMode, MVT::f64, Custom);
 
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);
    }
 
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
    setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
    setOperationAction(ISD::SELECT, MVT::f64, Custom);
    setOperationAction(ISD::BR_CC, MVT::f64, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
    setTruncStoreAction(MVT::f64, MVT::f32, Expand);
    setOperationAction(FPOpToExpand, MVT::f64, Expand);
    setOperationAction(FPOpToLibCall, MVT::f64, LibCall);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
    setTruncStoreAction(MVT::f64, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
    setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
    setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
    setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
    setOperationAction(ISD::FP_TO_BF16, MVT::f64,
                       Subtarget.isSoftFPABI() ? LibCall : Custom);
    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
    setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
  }
 
  if (Subtarget.is64Bit()) {
    setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
                        ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
                       MVT::i32, Custom);
    setOperationAction(ISD::LROUND, MVT::i32, Custom);
  } else if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
    // i64 is not a legal type on RV32, so these are custom expanded in
    // ReplaceNodeResults: f16 sources are converted to i32 and extended, other
    // FP sources fall back to the default libcall expansion.
    setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
                        ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
                       MVT::i64, Custom);
  }
 
  if (Subtarget.hasStdExtFOrZfinx()) {
    setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
                       Custom);
 
    // f16/bf16 require custom handling.
    setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, XLenVT,
                       Custom);
    setOperationAction({ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, XLenVT,
                       Custom);
 
    setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
    setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
    setOperationAction(ISD::GET_FPENV, XLenVT, Custom);
    setOperationAction(ISD::SET_FPENV, XLenVT, Custom);
    setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
    setOperationAction(ISD::GET_FPMODE, XLenVT, Custom);
    setOperationAction(ISD::SET_FPMODE, XLenVT, Custom);
    setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
  }
 
  setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
                      ISD::JumpTable},
                     XLenVT, Custom);
 
  setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
 
  if (Subtarget.is64Bit())
    setOperationAction(ISD::Constant, MVT::i64, Custom);
 
  // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
  // Unfortunately this can't be determined just from the ISA naming string.
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
                     Subtarget.is64Bit() ? Legal : Custom);
  setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64,
                     Subtarget.is64Bit() ? Legal : Custom);
 
  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
    setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
  }
 
  setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  if (Subtarget.is64Bit())
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
 
  if (Subtarget.hasVendorXMIPSCBOP())
    setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
  else
    setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
 
  if (Subtarget.hasStdExtZalrsc()) {
    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
    if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
      setMinCmpXchgSizeInBits(8);
    else
      setMinCmpXchgSizeInBits(32);
  } else if (Subtarget.hasForcedAtomics()) {
    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
  } else {
    setMaxAtomicSizeInBitsSupported(0);
  }
 
  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
 
  setBooleanContents(ZeroOrOneBooleanContent);
 
  if (getTargetMachine().getTargetTriple().isOSLinux()) {
    // Custom lowering of llvm.clear_cache.
    setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
  }
 
  if (Subtarget.hasVInstructions()) {
    setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
    setOperationAction(ISD::VSCALE, XLenVT, Custom);
 
    // RVV intrinsics may have illegal operands.
    // We also need to custom legalize vmv.x.s.
    setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
                        ISD::INTRINSIC_VOID},
                       {MVT::i8, MVT::i16}, Custom);
    if (Subtarget.is64Bit())
      setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
                         MVT::i32, Custom);
    else
      setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
                         MVT::i64, Custom);
 
    setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
                       MVT::Other, Custom);
 
    static const unsigned IntegerVPOps[] = {
        ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
        ISD::VP_UREM,        ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
        ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
        ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
        ISD::VP_MERGE,
        ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
        ISD::VP_CTTZ_ELTS,   ISD::VP_CTTZ_ELTS_ZERO_POISON};
 
    static const unsigned FloatingPointVPOps[] = {
        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
        ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
        ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
 
    static const unsigned IntegerVecReduceOps[] = {
        ISD::VECREDUCE_ADD,  ISD::VECREDUCE_AND,  ISD::VECREDUCE_OR,
        ISD::VECREDUCE_XOR,  ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
        ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
 
    static const unsigned FloatingPointVecReduceOps[] = {
        ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
        ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
 
    static const unsigned FloatingPointLibCallOps[] = {
        ISD::FREM,  ISD::FPOW,   ISD::FCOS, ISD::FSIN,  ISD::FSINCOS, ISD::FEXP,
        ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
 
    if (!Subtarget.is64Bit()) {
      // We must custom-lower certain vXi64 operations on RV32 due to the vector
      // element type being illegal.
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                         MVT::i64, Custom);
 
      setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
 
      setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
                          ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
                          ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
                          ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
                         MVT::i64, Custom);
    }
 
    for (MVT VT : BoolVecVTs) {
      if (!isTypeLegal(VT))
        continue;
 
      setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
 
      // Mask VTs are custom-expanded into a series of standard nodes
      setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
                          ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
                          ISD::SCALAR_TO_VECTOR},
                         VT, Custom);
 
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
                         Custom);
 
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction({ISD::SELECT_CC, ISD::VSELECT}, VT,
                         Expand);
      setOperationAction(ISD::VP_MERGE, VT, Custom);
 
      setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
                          ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
                         VT, Custom);
 
      setOperationAction(
          {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
          Custom);
 
      setOperationAction(
          {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
          Custom);
 
      // RVV has native int->float & float->int conversions where the
      // element type sizes are within one power-of-two of each other. Any
      // wider distances between type sizes have to be lowered as sequences
      // which progressively narrow the gap in stages.
      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
                          ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
                          ISD::STRICT_FP_TO_UINT},
                         VT, Custom);
      setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                         Custom);
 
      // Expand all extending loads to types larger than this, and truncating
      // stores from types larger than this.
      for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
        setTruncStoreAction(VT, OtherVT, Expand);
        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                         OtherVT, Expand);
      }
 
      setVectorInterleaveAction(
          {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
          {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
 
      setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
 
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
      setOperationPromotedToType(
          {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
          MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
    }
 
    for (MVT VT : IntVecVTs) {
      if (!isTypeLegal(VT))
        continue;
 
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
 
      // Vectors implement MULHS/MULHU.
      setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
 
      // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
      if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
        setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
 
      setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
                         Legal);
 
      if (Subtarget.hasStdExtZvabd()) {
        setOperationAction(ISD::ABS, VT, Legal);
        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Legal);
      } else {
        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
      }
 
      // Custom-lower extensions and truncations from/to mask types.
      setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
                         VT, Custom);
 
      // RVV has native int->float & float->int conversions where the
      // element type sizes are within one power-of-two of each other. Any
      // wider distances between type sizes have to be lowered as sequences
      // which progressively narrow the gap in stages.
      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
                          ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
                          ISD::STRICT_FP_TO_UINT},
                         VT, Custom);
      setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                         Custom);
      setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
                          ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
                          ISD::SSUBSAT, ISD::USUBSAT},
                         VT, Legal);
 
      // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
      // nodes which truncate by one power of two at a time.
      setOperationAction(
          {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
          Custom);
 
      // Custom-lower insert/extract operations to simplify patterns.
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
                         Custom);
 
      // Custom-lower reduction operations to set up the corresponding custom
      // nodes' operands.
      setOperationAction(IntegerVecReduceOps, VT, Custom);
 
      setOperationAction(IntegerVPOps, VT, Custom);
 
      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
 
      setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
                         VT, Custom);
 
      setOperationAction(
          {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
           ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
          VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                          ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
                         VT, Custom);
 
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
 
      setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
 
      for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
        setTruncStoreAction(VT, OtherVT, Expand);
        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                         OtherVT, Expand);
      }
 
      setVectorInterleaveAction(
          {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
          {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
 
      setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
                         VT, Custom);
 
      if (Subtarget.hasStdExtZvkb()) {
        setOperationAction(ISD::BSWAP, VT, Legal);
      } else {
        setOperationAction(ISD::BSWAP, VT, Expand);
        setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
      }
 
      if (Subtarget.hasStdExtZvbb()) {
        setOperationAction(ISD::BITREVERSE, VT, Legal);
      } else {
        setOperationAction(ISD::BITREVERSE, VT, Expand);
        setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
 
        // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
        // range of f32.
        EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        if (isTypeLegal(FloatVT)) {
          setOperationAction(
              {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
              Custom);
        }
      }
 
      if (VT.getVectorElementType() == MVT::i64) {
        if (Subtarget.hasStdExtZvbc())
          setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
      } else {
        if (Subtarget.hasStdExtZvbc32e()) {
          setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
        } else if (Subtarget.hasStdExtZvbc()) {
          // Promote to i64 if the lmul is small enough.
          // FIXME: Split if necessary to widen.
          // FIXME: Promote clmulh directly without legalizing to clmul first.
          MVT I64VecVT = MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
          if (isTypeLegal(I64VecVT))
            setOperationAction(ISD::CLMUL, VT, Custom);
        }
      }
 
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
      setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
                          ISD::MASKED_SREM},
                         VT, Legal);
    }
 
    for (MVT VT : VecTupleVTs) {
      if (!isTypeLegal(VT))
        continue;
 
      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
    }
 
    // Expand various CCs to best match the RVV ISA, which natively supports UNE
    // but no other unordered comparisons, and supports all ordered comparisons
    // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
    // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
    // and we pattern-match those back to the "original", swapping operands once
    // more. This way we catch both operations and both "vf" and "fv" forms with
    // fewer patterns.
    static const ISD::CondCode VFPCCToExpand[] = {
        ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
        ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
        ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
    };
 
    // TODO: support more ops.
    static const unsigned ZvfhminZvfbfminPromoteOps[] = {
        ISD::FMINNUM,
        ISD::FMAXNUM,
        ISD::FMINIMUMNUM,
        ISD::FMAXIMUMNUM,
        ISD::FADD,
        ISD::FSUB,
        ISD::FMUL,
        ISD::FMA,
        ISD::FDIV,
        ISD::FSQRT,
        ISD::FCEIL,
        ISD::FTRUNC,
        ISD::FFLOOR,
        ISD::FROUND,
        ISD::FROUNDEVEN,
        ISD::FRINT,
        ISD::FNEARBYINT,
        ISD::IS_FPCLASS,
        ISD::SETCC,
        ISD::FMAXIMUM,
        ISD::FMINIMUM,
        ISD::STRICT_FADD,
        ISD::STRICT_FSUB,
        ISD::STRICT_FMUL,
        ISD::STRICT_FDIV,
        ISD::STRICT_FSQRT,
        ISD::STRICT_FMA,
        ISD::VECREDUCE_FADD,
        ISD::VECREDUCE_FMIN,
        ISD::VECREDUCE_FMAX,
        ISD::VECREDUCE_FMINIMUM,
        ISD::VECREDUCE_FMAXIMUM,
        ISD::FCANONICALIZE};
 
    // TODO: Make more of these ops legal.
    static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
                                                ISD::FSQRT,
                                                ISD::FCEIL,
                                                ISD::FTRUNC,
                                                ISD::FFLOOR,
                                                ISD::FROUND,
                                                ISD::FROUNDEVEN,
                                                ISD::FRINT,
                                                ISD::FNEARBYINT,
                                                ISD::STRICT_FDIV,
                                                ISD::STRICT_FSQRT,
                                                ISD::VECREDUCE_FADD,
                                                ISD::VECREDUCE_FMIN,
                                                ISD::VECREDUCE_FMAX,
                                                ISD::VECREDUCE_FMINIMUM,
                                                ISD::VECREDUCE_FMAXIMUM};
 
    // TODO: support more vp ops.
    static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
        ISD::VP_REDUCE_FMIN,
        ISD::VP_REDUCE_FMAX,
        ISD::VP_REDUCE_FMINIMUM,
        ISD::VP_REDUCE_FMAXIMUM};
 
    // Sets common operation actions on RVV floating-point vector types.
    const auto SetCommonVFPActions = [&](MVT VT) {
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
      // sizes are within one power-of-two of each other. Therefore conversions
      // between vXf16 and vXf64 must be lowered as sequences which convert via
      // vXf32.
      setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
      setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
      // Custom-lower insert/extract operations to simplify patterns.
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
                         Custom);
      // Expand various condition codes (explained above).
      setCondCodeAction(VFPCCToExpand, VT, Expand);
 
      setOperationAction(
          {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
          Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
 
      setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
                          ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
                          ISD::IS_FPCLASS},
                         VT, Custom);
 
      setOperationAction(FloatingPointVecReduceOps, VT, Custom);
 
      // Expand FP operations that need libcalls.
      setOperationAction(FloatingPointLibCallOps, VT, Expand);
 
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
 
      setOperationAction(ISD::FCOPYSIGN, VT, Legal);
 
      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
 
      setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
                         VT, Custom);
 
      setOperationAction(
          {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
           ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
          VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
 
      setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                          ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
                         VT, Custom);
 
      setVectorInterleaveAction(
          {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
          {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
 
      setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
                          ISD::VECTOR_SPLICE_RIGHT},
                         VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
      setOperationAction(FloatingPointVPOps, VT, Custom);
 
      setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
                         Custom);
      setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                          ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
                         VT, Legal);
      setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
                          ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
                          ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
                          ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
                         VT, Custom);
 
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
    };
 
    // Sets common extload/truncstore actions on RVV floating-point vector
    // types.
    const auto SetCommonVFPExtLoadTruncStoreActions =
        [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
          for (auto SmallVT : SmallerVTs) {
            setTruncStoreAction(VT, SmallVT, Expand);
            setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
          }
        };
 
    // Sets common actions for f16 and bf16 for when there's only
    // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
    const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
      setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
      setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
                         Custom);
      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
      setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
      setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
                         Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
                          ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
                          ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
                          ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
                         VT, Custom);
      setVectorInterleaveAction(
          {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
          {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
      MVT EltVT = VT.getVectorElementType();
      if (isTypeLegal(EltVT))
        setOperationAction({ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
                           VT, Custom);
      else
        setOperationAction(ISD::SPLAT_VECTOR, EltVT, Custom);
      setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                          ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
                          ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                          ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
                          ISD::VP_SCATTER},
                         VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      setOperationAction(ISD::FNEG, VT, Expand);
      setOperationAction(ISD::FABS, VT, Expand);
      setOperationAction(ISD::FCOPYSIGN, VT, Expand);
 
      // Expand FP operations that need libcalls.
      setOperationAction(FloatingPointLibCallOps, VT, Expand);
 
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
 
      // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
      if (getLMUL(VT) == RISCVVType::LMUL_8) {
        setOperationAction(ZvfhminZvfbfminPromoteOps, VT, Custom);
        setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
      } else {
        MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
        setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
      }
    };
 
    // Sets common actions for zvfbfa, some of instructions are supported
    // natively so that we don't need to promote them.
    const auto SetZvfbfaActions = [&](MVT VT) {
      setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
      setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
                         Custom);
      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
      setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
      setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
                         Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                          ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                          ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
                          ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
                          ISD::VECTOR_COMPRESS},
                         VT, Custom);
      setVectorInterleaveAction(
          {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
          {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
      setOperationAction(
          {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
          Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
      setOperationAction(ISD::IS_FPCLASS, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
      setOperationAction(ISD::FCOPYSIGN, VT, Legal);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                          ISD::STRICT_FMA},
                         VT, Legal);
      setCondCodeAction(VFPCCToExpand, VT, Expand);
 
      setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                          ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
                          ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                          ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
                          ISD::VP_SCATTER},
                         VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      // Expand FP operations that need libcalls.
      setOperationAction(FloatingPointLibCallOps, VT, Expand);
 
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
 
      // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
      if (getLMUL(VT) == RISCVVType::LMUL_8) {
        setOperationAction(ZvfbfaPromoteOps, VT, Custom);
        setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
      } else {
        MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
        setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
      }
    };
 
    if (Subtarget.hasVInstructionsF16()) {
      for (MVT VT : F16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonVFPActions(VT);
      }
    } else if (Subtarget.hasVInstructionsF16Minimal()) {
      for (MVT VT : F16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonPromoteToF32Actions(VT);
      }
    }
 
    if (Subtarget.hasVInstructionsBF16()) {
      for (MVT VT : BF16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetZvfbfaActions(VT);
      }
    } else if (Subtarget.hasVInstructionsBF16Minimal()) {
      for (MVT VT : BF16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonPromoteToF32Actions(VT);
      }
    }
 
    if (Subtarget.hasStdExtZvfofp8min()) {
      for (MVT VT : BF16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, VT, Custom);
      }
    }
 
    if (Subtarget.hasVInstructionsF32()) {
      for (MVT VT : F32VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonVFPActions(VT);
        SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
        SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
      }
    }
 
    if (Subtarget.hasVInstructionsF64()) {
      for (MVT VT : F64VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonVFPActions(VT);
        SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
        SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
        SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
      }
    }
 
    if (Subtarget.useRVVForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        if (!useRVVForFixedLengthVectorVT(VT))
          continue;
 
        // By default everything must be expanded.
        for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
          setOperationAction(Op, VT, Expand);
        for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
          setTruncStoreAction(VT, OtherVT, Expand);
          setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                           OtherVT, Expand);
        }
 
        // Custom lower fixed vector undefs to scalable vector undefs to avoid
        // expansion to a build_vector of 0s.
        setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
 
        // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
        setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
                           Custom);
 
        setOperationAction(
            {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
            Custom);
 
        setVectorInterleaveAction(
            {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
            {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
 
        setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                           VT, Custom);
 
        setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
 
        setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
 
        setOperationAction(ISD::SETCC, VT, Custom);
 
        setOperationAction(ISD::SELECT, VT, Custom);
 
        setOperationAction(
            {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
            Custom);
 
        setOperationAction(ISD::BITCAST, VT, Custom);
 
        setOperationAction(
            {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
            Custom);
 
        setOperationAction(
            {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
            Custom);
 
        setOperationAction(
            {
                ISD::SINT_TO_FP,
                ISD::UINT_TO_FP,
                ISD::FP_TO_SINT,
                ISD::FP_TO_UINT,
                ISD::STRICT_SINT_TO_FP,
                ISD::STRICT_UINT_TO_FP,
                ISD::STRICT_FP_TO_SINT,
                ISD::STRICT_FP_TO_UINT,
            },
            VT, Custom);
        setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                           Custom);
 
        setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
 
        // Operations below are different for between masks and other vectors.
        if (VT.getVectorElementType() == MVT::i1) {
          setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Custom);
 
          setOperationAction(ISD::VP_MERGE, VT, Custom);
 
          setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
          setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
          setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
                             Custom);
          continue;
        }
 
        // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
        // it before type legalization for i64 vectors on RV32. It will then be
        // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
        // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
        // improvements first.
        if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
          setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
          setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
 
          // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
          setOperationAction(ISD::BUILD_VECTOR, MVT::i64, Custom);
        }
 
        setOperationAction(
            {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
 
        setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
                            ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                            ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
                            ISD::VP_SCATTER},
                           VT, Custom);
        setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
        setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
                            ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
                            ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
                           VT, Custom);
 
        setOperationAction(
            {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
 
        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
 
        // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
        if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
          setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
 
        setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
                            ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
                            ISD::SSUBSAT, ISD::USUBSAT},
                           VT, Custom);
 
        setOperationAction(ISD::VSELECT, VT, Custom);
 
        setOperationAction(
            {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
 
        // Custom-lower reduction operations to set up the corresponding custom
        // nodes' operands.
        setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
                            ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
                            ISD::VECREDUCE_UMIN},
                           VT, Custom);
 
        setOperationAction(IntegerVPOps, VT, Custom);
 
        if (Subtarget.hasStdExtZvkb())
          setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
 
        if (Subtarget.hasStdExtZvbb()) {
          setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
                              ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
                             VT, Custom);
        } else {
          // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
          // range of f32.
          EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
          if (isTypeLegal(FloatVT))
            setOperationAction(
                {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
                Custom);
        }
 
        if (VT.getVectorElementType() == MVT::i64) {
          if (Subtarget.hasStdExtZvbc())
            setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
        } else {
          if (Subtarget.hasStdExtZvbc32e()) {
            setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
          } else if (Subtarget.hasStdExtZvbc()) {
            // Promote to i64 as is done for scalable vectors.
            MVT I64VecVT =
                MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
            if (I64VecVT.isValid() && useRVVForFixedLengthVectorVT(I64VecVT))
              setOperationAction(ISD::CLMUL, VT, Custom);
          }
        }
 
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
        setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV,
                            ISD::MASKED_UREM, ISD::MASKED_SREM},
                           VT, Custom);
      }
 
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
        // There are no extending loads or truncating stores.
        for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
          setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
          setTruncStoreAction(VT, InnerVT, Expand);
        }
 
        if (!useRVVForFixedLengthVectorVT(VT))
          continue;
 
        // By default everything must be expanded.
        for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
          setOperationAction(Op, VT, Expand);
 
        // Custom lower fixed vector undefs to scalable vector undefs to avoid
        // expansion to a build_vector of 0s.
        setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
 
        setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                            ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                            ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
                            ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
                           VT, Custom);
        setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
        setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
        setVectorInterleaveAction(
            {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
            {2, 3, 4, 5, 6, 7, 8}, VT, Custom);
 
        setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                            ISD::MGATHER, ISD::MSCATTER},
                           VT, Custom);
        setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
                            ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                            ISD::EXPERIMENTAL_VP_STRIDED_STORE},
                           VT, Custom);
        setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
        setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
        setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
                           Custom);
 
        setOperationAction(ISD::BITCAST, VT, Custom);
 
        if (VT.getVectorElementType() == MVT::f16 &&
            !Subtarget.hasVInstructionsF16()) {
          setOperationAction(
              {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
              Custom);
          setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
          setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
          if (Subtarget.hasStdExtZfhmin()) {
            setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
          } else {
            // We need to custom legalize f16 build vectors if Zfhmin isn't
            // available.
            setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
          }
          setOperationAction(ISD::FNEG, VT, Expand);
          setOperationAction(ISD::FABS, VT, Expand);
          setOperationAction(ISD::FCOPYSIGN, VT, Expand);
          MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
          // Don't promote f16 vector operations to f32 if f32 vector type is
          // not legal.
          // Custom lower maximum LMUL case to split to 2 half LMUL operations.
          // TODO: Support more operations.
          if (!isTypeLegal(F32VecVT)) {
            setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
                                ISD::VECREDUCE_FMAXIMUM,
                                ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
                               VT, Custom);
            setOperationAction(ISD::SETCC, VT, Custom);
            continue;
          }
          setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
          setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
          continue;
        }
 
        if (VT.getVectorElementType() == MVT::bf16) {
          setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
          setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
          if (Subtarget.hasStdExtZvfofp8min())
            setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, VT, Custom);
          if (Subtarget.hasStdExtZfbfmin()) {
            setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
          } else {
            // We need to custom legalize bf16 build vectors if Zfbfmin isn't
            // available.
            setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
          }
          if (Subtarget.hasVInstructionsBF16()) {
            setOperationAction(ZvfbfaOps, VT, Custom);
            setCondCodeAction(VFPCCToExpand, VT, Expand);
          }
          setOperationAction(
              {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
              Custom);
          MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
          // Don't promote bf16 vector operations to f32 if f32 vector type is
          // not legal.
          // Custom lower maximum LMUL case to split to 2 half LMUL operations.
          // TODO: Support more operations.
          if (!isTypeLegal(F32VecVT)) {
            setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
                                ISD::VECREDUCE_FMAXIMUM,
                                ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
                               VT, Custom);
            setOperationAction(ISD::SETCC, VT, Custom);
            continue;
          }
 
          if (Subtarget.hasVInstructionsBF16())
            setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
          else
            setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
          setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
          continue;
        }
 
        setOperationAction({ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
                           Custom);
 
        setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
                            ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
                            ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
                            ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
                            ISD::FMAXIMUM, ISD::FMINIMUM},
                           VT, Custom);
 
        setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
                            ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
                            ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
                            ISD::FNEARBYINT, ISD::FCANONICALIZE},
                           VT, Custom);
 
        setCondCodeAction(VFPCCToExpand, VT, Expand);
 
        setOperationAction(ISD::SETCC, VT, Custom);
        setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
 
        setOperationAction(FloatingPointVecReduceOps, VT, Custom);
 
        setOperationAction(FloatingPointVPOps, VT, Custom);
 
        setOperationAction(
            {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
             ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
             ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
             ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
             ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
            VT, Custom);
      }
 
      // Custom-legalize bitcasts from fixed-length vectors to scalar types.
      setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32}, Custom);
      if (Subtarget.is64Bit())
        setOperationAction(ISD::BITCAST, MVT::i64, Custom);
      if (Subtarget.hasStdExtZfhminOrZhinxmin())
        setOperationAction(ISD::BITCAST, MVT::f16, Custom);
      if (Subtarget.hasStdExtZfbfmin())
        setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
      if (Subtarget.hasStdExtFOrZfinx())
        setOperationAction(ISD::BITCAST, MVT::f32, Custom);
      if (Subtarget.hasStdExtDOrZdinx())
        setOperationAction(ISD::BITCAST, MVT::f64, Custom);
    }
  }
 
  if (Subtarget.hasStdExtZaamo())
    setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
 
  if (Subtarget.hasForcedAtomics()) {
    // Force __sync libcalls to be emitted for atomic rmw/cas operations.
    setOperationAction(
        {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
         ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
         ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
         ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
        XLenVT, LibCall);
  }
 
  if (Subtarget.hasVendorXTHeadMemIdx()) {
    for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
      setIndexedLoadAction(im, MVT::i8, Legal);
      setIndexedStoreAction(im, MVT::i8, Legal);
      setIndexedLoadAction(im, MVT::i16, Legal);
      setIndexedStoreAction(im, MVT::i16, Legal);
      setIndexedLoadAction(im, MVT::i32, Legal);
      setIndexedStoreAction(im, MVT::i32, Legal);
 
      if (Subtarget.is64Bit()) {
        setIndexedLoadAction(im, MVT::i64, Legal);
        setIndexedStoreAction(im, MVT::i64, Legal);
      }
    }
  }
 
  if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
    setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
    setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
    setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
 
    setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
    setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
    setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
  }
 
  // zve32x is broken for partial_reduce_umla, but let's not make it worse.
  if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
    static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                      ISD::PARTIAL_REDUCE_UMLA,
                                      ISD::PARTIAL_REDUCE_SUMLA};
    setPartialReduceMLAAction(MLAOps, MVT::nxv1i32, MVT::nxv4i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv2i32, MVT::nxv8i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom);
 
    // An i64 accumulator is handled by performing an i32 vdot4a* and widening
    // the result to i64 (see lowerPARTIAL_REDUCE_MLA).
    setPartialReduceMLAAction(MLAOps, MVT::nxv1i64, MVT::nxv8i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv4i64, MVT::nxv32i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv8i64, MVT::nxv64i8, Custom);
 
    if (Subtarget.useRVVForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        if ((VT.getVectorElementType() != MVT::i32 &&
             VT.getVectorElementType() != MVT::i64) ||
            !useRVVForFixedLengthVectorVT(VT))
          continue;
        ElementCount EC = VT.getVectorElementCount();
        unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4;
        MVT ArgVT = MVT::getVectorVT(MVT::i8, EC * Scale);
        setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom);
      }
    }
  }
 
  // Customize load and store operation for bf16 if zfh isn't enabled.
  if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
    setOperationAction(ISD::LOAD, MVT::bf16, Custom);
    setOperationAction(ISD::STORE, MVT::bf16, Custom);
  }
 
  // Function alignments.
  const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
  setMinFunctionAlignment(FunctionAlignment);
  // Set preferred alignments.
  setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
  setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
 
  setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
                       ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
                       ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
                       ISD::SRA});
  setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
 
  if (Subtarget.hasStdExtFOrZfinx())
    setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
 
  // Allow scalar min/max to be combined with vector reductions.
  if (Subtarget.hasVInstructions())
    setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
  if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
    setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
 
  if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
      Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
    setTargetDAGCombine(ISD::TRUNCATE);
 
  if (Subtarget.hasStdExtZbkb())
    setTargetDAGCombine(ISD::BITREVERSE);
 
  if (Subtarget.hasStdExtFOrZfinx())
    setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                         ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
  if (Subtarget.hasVInstructions())
    setTargetDAGCombine({ISD::FCOPYSIGN,
                         ISD::VECTOR_MATCH,
                         ISD::MGATHER,
                         ISD::MSCATTER,
                         ISD::MLOAD,
                         ISD::VP_GATHER,
                         ISD::VP_SCATTER,
                         ISD::SRL,
                         ISD::SHL,
                         ISD::STORE,
                         ISD::SPLAT_VECTOR,
                         ISD::BUILD_VECTOR,
                         ISD::CONCAT_VECTORS,
                         ISD::VP_STORE,
                         ISD::EXPERIMENTAL_VP_REVERSE,
                         ISD::SDIV,
                         ISD::UDIV,
                         ISD::SREM,
                         ISD::UREM,
                         ISD::INSERT_VECTOR_ELT,
                         ISD::ABS,
                         ISD::ABS_MIN_POISON,
                         ISD::CTPOP,
                         ISD::VECTOR_SHUFFLE,
                         ISD::FMA,
                         ISD::VSELECT,
                         ISD::VECREDUCE_ADD,
                         ISD::VECTOR_SPLICE_RIGHT});
 
  if (Subtarget.hasStdExtZvzip())
    setTargetDAGCombine({ISD::VECTOR_INTERLEAVE});
 
  if (Subtarget.hasVendorXTHeadMemPair())
    setTargetDAGCombine({ISD::LOAD, ISD::STORE});
  if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
    setTargetDAGCombine(ISD::BITCAST);
 
  setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
 
  setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
 
  setJumpIsExpensive(Subtarget.isJumpExpensive());
 
  // Disable strict node mutation.
  IsStrictFPEnabled = true;
  EnableExtLdPromotion = true;
 
  // Let the subtarget decide if a predictable select is more expensive than the
  // corresponding branch. This information is used in CGP/SelectOpt to decide
  // when to convert selects into branches.
  PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
 
  MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
  MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
 
  MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
  MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
  MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
 
  MaxStoresPerMemmoveOptSize =
      Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
  MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
 
  MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
  MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
}
 
TargetLoweringBase::LegalizeTypeAction
RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
  if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
    if (VT == MVT::v2i16 || VT == MVT::v4i8)
      return TypeWidenVector;
 
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
                                            LLVMContext &Context,
                                            EVT VT) const {
  if (!VT.isVector())
    return getPointerTy(DL);
  if (Subtarget.hasVInstructions() &&
      (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
    return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
  return VT.changeVectorElementTypeToInteger();
}
 
TargetLoweringBase::CondMergingParams
RISCVTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
                                                   const Value *LHS,
                                                   const Value *RHS,
                                                   const Function *F) const {
  if (F->hasOptSize())
    return TargetLowering::getJumpConditionMergingParams(Opc, LHS, RHS, F);
 
  // Merging conditions eliminates a branch, so the budget we are willing to
  // spend eagerly computing the RHS condition should scale with how expensive a
  // mispredicted branch is. A branch only costs the full penalty when actually
  // mispredicted, so scale it down by an assumed misprediction rate (~25%).
  int BaseCost = Subtarget.getMispredictionPenalty() / 4;
  if (BrMergingBaseCostThresh.getNumOccurrences() > 1)
    BaseCost = BrMergingBaseCostThresh;
 
  return {BaseCost, BrMergingLikelyBias, BrMergingUnlikelyBias};
}
 
MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
  return Subtarget.getXLenVT();
}
 
bool RISCVTargetLowering::isProfitableToFoldVScaleAdd(SDValue N) const {
  return N.hasOneUse();
}
 
// Return false if we can lower get_vector_length to a vsetvli intrinsic.
bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
                                                      unsigned VF,
                                                      bool IsScalable) const {
  if (!Subtarget.hasVInstructions())
    return true;
 
  if (!IsScalable)
    return true;
 
  if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
    return true;
 
  // Don't allow VF=1 if those types are't legal.
  if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
    return true;
 
  // VLEN=32 support is incomplete.
  if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
    return true;
 
  // The maximum VF is for the smallest element width with LMUL=8.
  // VF must be a power of 2.
  unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
  return VF > MaxVF || !isPowerOf2_32(VF);
}
 
void RISCVTargetLowering::getTgtMemIntrinsic(
    SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
    MachineFunction &MF, unsigned Intrinsic) const {
  IntrinsicInfo Info;
  auto &DL = I.getDataLayout();
 
  auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
                                 bool IsUnitStrided, bool UsePtrVal = false) {
    Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
    // We can't use ptrVal if the intrinsic can access memory before the
    // pointer. This means we can't use it for strided or indexed intrinsics.
    if (UsePtrVal)
      Info.ptrVal = I.getArgOperand(PtrOp);
    else
      Info.fallbackAddressSpace =
          I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
    Type *MemTy;
    if (IsStore) {
      // Store value is the first operand.
      MemTy = I.getArgOperand(0)->getType();
    } else {
      // Use return type. If it's segment load, return type is a struct.
      MemTy = I.getType();
      if (MemTy->isStructTy())
        MemTy = MemTy->getStructElementType(0);
    }
    if (!IsUnitStrided)
      MemTy = MemTy->getScalarType();
 
    Info.memVT = getValueType(DL, MemTy);
    if (MemTy->isTargetExtTy()) {
      // RISC-V vector tuple type's alignment type should be its element type.
      if (cast<TargetExtType>(MemTy)->getName() == "riscv.vector.tuple")
        MemTy = Type::getIntNTy(
            MemTy->getContext(),
            1 << cast<ConstantInt>(I.getArgOperand(I.arg_size() - 1))
                     ->getZExtValue());
      Info.align = DL.getABITypeAlign(MemTy);
    } else {
      Info.align = Align(DL.getTypeStoreSize(MemTy->getScalarType()));
    }
    Info.size = MemoryLocation::UnknownSize;
    Info.flags |=
        IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
    Infos.push_back(Info);
  };
 
  if (I.hasMetadata(LLVMContext::MD_nontemporal))
    Info.flags |= MachineMemOperand::MONonTemporal;
 
  Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
  switch (Intrinsic) {
  default:
    return;
  case Intrinsic::riscv_masked_atomicrmw_xchg:
  case Intrinsic::riscv_masked_atomicrmw_add:
  case Intrinsic::riscv_masked_atomicrmw_sub:
  case Intrinsic::riscv_masked_atomicrmw_nand:
  case Intrinsic::riscv_masked_atomicrmw_max:
  case Intrinsic::riscv_masked_atomicrmw_min:
  case Intrinsic::riscv_masked_atomicrmw_umax:
  case Intrinsic::riscv_masked_atomicrmw_umin:
  case Intrinsic::riscv_masked_cmpxchg:
    // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
    // narrow atomic operation. These will be expanded to an LR/SC loop that
    // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
    // will be used to modify the appropriate part of the 4 byte data and
    // preserve the rest.
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.align = Align(4);
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                 MachineMemOperand::MOVolatile;
    Infos.push_back(Info);
    return;
  case Intrinsic::riscv_seg2_load_mask:
  case Intrinsic::riscv_seg3_load_mask:
  case Intrinsic::riscv_seg4_load_mask:
  case Intrinsic::riscv_seg5_load_mask:
  case Intrinsic::riscv_seg6_load_mask:
  case Intrinsic::riscv_seg7_load_mask:
  case Intrinsic::riscv_seg8_load_mask:
  case Intrinsic::riscv_sseg2_load_mask:
  case Intrinsic::riscv_sseg3_load_mask:
  case Intrinsic::riscv_sseg4_load_mask:
  case Intrinsic::riscv_sseg5_load_mask:
  case Intrinsic::riscv_sseg6_load_mask:
  case Intrinsic::riscv_sseg7_load_mask:
  case Intrinsic::riscv_sseg8_load_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_seg2_store_mask:
  case Intrinsic::riscv_seg3_store_mask:
  case Intrinsic::riscv_seg4_store_mask:
  case Intrinsic::riscv_seg5_store_mask:
  case Intrinsic::riscv_seg6_store_mask:
  case Intrinsic::riscv_seg7_store_mask:
  case Intrinsic::riscv_seg8_store_mask:
    // Operands are (vec, ..., vec, ptr, mask, vl)
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_sseg2_store_mask:
  case Intrinsic::riscv_sseg3_store_mask:
  case Intrinsic::riscv_sseg4_store_mask:
  case Intrinsic::riscv_sseg5_store_mask:
  case Intrinsic::riscv_sseg6_store_mask:
  case Intrinsic::riscv_sseg7_store_mask:
  case Intrinsic::riscv_sseg8_store_mask:
    // Operands are (vec, ..., vec, ptr, offset, mask, vl)
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlm:
    SetRVVLoadStoreInfo(/*PtrOp*/ 0,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ true,
                        /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vle:
  case Intrinsic::riscv_vle_mask:
  case Intrinsic::riscv_vleff:
  case Intrinsic::riscv_vleff_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ true,
                        /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vsm:
  case Intrinsic::riscv_vse:
  case Intrinsic::riscv_vse_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ true,
                        /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlse:
  case Intrinsic::riscv_vlse_mask:
  case Intrinsic::riscv_vloxei:
  case Intrinsic::riscv_vloxei_mask:
  case Intrinsic::riscv_vluxei:
  case Intrinsic::riscv_vluxei_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vsse:
  case Intrinsic::riscv_vsse_mask:
  case Intrinsic::riscv_vsoxei:
  case Intrinsic::riscv_vsoxei_mask:
  case Intrinsic::riscv_vsuxei:
  case Intrinsic::riscv_vsuxei_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vlseg2:
  case Intrinsic::riscv_vlseg3:
  case Intrinsic::riscv_vlseg4:
  case Intrinsic::riscv_vlseg5:
  case Intrinsic::riscv_vlseg6:
  case Intrinsic::riscv_vlseg7:
  case Intrinsic::riscv_vlseg8:
  case Intrinsic::riscv_vlseg2ff:
  case Intrinsic::riscv_vlseg3ff:
  case Intrinsic::riscv_vlseg4ff:
  case Intrinsic::riscv_vlseg5ff:
  case Intrinsic::riscv_vlseg6ff:
  case Intrinsic::riscv_vlseg7ff:
  case Intrinsic::riscv_vlseg8ff:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlseg2_mask:
  case Intrinsic::riscv_vlseg3_mask:
  case Intrinsic::riscv_vlseg4_mask:
  case Intrinsic::riscv_vlseg5_mask:
  case Intrinsic::riscv_vlseg6_mask:
  case Intrinsic::riscv_vlseg7_mask:
  case Intrinsic::riscv_vlseg8_mask:
  case Intrinsic::riscv_vlseg2ff_mask:
  case Intrinsic::riscv_vlseg3ff_mask:
  case Intrinsic::riscv_vlseg4ff_mask:
  case Intrinsic::riscv_vlseg5ff_mask:
  case Intrinsic::riscv_vlseg6ff_mask:
  case Intrinsic::riscv_vlseg7ff_mask:
  case Intrinsic::riscv_vlseg8ff_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlsseg2:
  case Intrinsic::riscv_vlsseg3:
  case Intrinsic::riscv_vlsseg4:
  case Intrinsic::riscv_vlsseg5:
  case Intrinsic::riscv_vlsseg6:
  case Intrinsic::riscv_vlsseg7:
  case Intrinsic::riscv_vlsseg8:
  case Intrinsic::riscv_vloxseg2:
  case Intrinsic::riscv_vloxseg3:
  case Intrinsic::riscv_vloxseg4:
  case Intrinsic::riscv_vloxseg5:
  case Intrinsic::riscv_vloxseg6:
  case Intrinsic::riscv_vloxseg7:
  case Intrinsic::riscv_vloxseg8:
  case Intrinsic::riscv_vluxseg2:
  case Intrinsic::riscv_vluxseg3:
  case Intrinsic::riscv_vluxseg4:
  case Intrinsic::riscv_vluxseg5:
  case Intrinsic::riscv_vluxseg6:
  case Intrinsic::riscv_vluxseg7:
  case Intrinsic::riscv_vluxseg8:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vlsseg2_mask:
  case Intrinsic::riscv_vlsseg3_mask:
  case Intrinsic::riscv_vlsseg4_mask:
  case Intrinsic::riscv_vlsseg5_mask:
  case Intrinsic::riscv_vlsseg6_mask:
  case Intrinsic::riscv_vlsseg7_mask:
  case Intrinsic::riscv_vlsseg8_mask:
  case Intrinsic::riscv_vloxseg2_mask:
  case Intrinsic::riscv_vloxseg3_mask:
  case Intrinsic::riscv_vloxseg4_mask:
  case Intrinsic::riscv_vloxseg5_mask:
  case Intrinsic::riscv_vloxseg6_mask:
  case Intrinsic::riscv_vloxseg7_mask:
  case Intrinsic::riscv_vloxseg8_mask:
  case Intrinsic::riscv_vluxseg2_mask:
  case Intrinsic::riscv_vluxseg3_mask:
  case Intrinsic::riscv_vluxseg4_mask:
  case Intrinsic::riscv_vluxseg5_mask:
  case Intrinsic::riscv_vluxseg6_mask:
  case Intrinsic::riscv_vluxseg7_mask:
  case Intrinsic::riscv_vluxseg8_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vsseg2:
  case Intrinsic::riscv_vsseg3:
  case Intrinsic::riscv_vsseg4:
  case Intrinsic::riscv_vsseg5:
  case Intrinsic::riscv_vsseg6:
  case Intrinsic::riscv_vsseg7:
  case Intrinsic::riscv_vsseg8:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vsseg2_mask:
  case Intrinsic::riscv_vsseg3_mask:
  case Intrinsic::riscv_vsseg4_mask:
  case Intrinsic::riscv_vsseg5_mask:
  case Intrinsic::riscv_vsseg6_mask:
  case Intrinsic::riscv_vsseg7_mask:
  case Intrinsic::riscv_vsseg8_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vssseg2:
  case Intrinsic::riscv_vssseg3:
  case Intrinsic::riscv_vssseg4:
  case Intrinsic::riscv_vssseg5:
  case Intrinsic::riscv_vssseg6:
  case Intrinsic::riscv_vssseg7:
  case Intrinsic::riscv_vssseg8:
  case Intrinsic::riscv_vsoxseg2:
  case Intrinsic::riscv_vsoxseg3:
  case Intrinsic::riscv_vsoxseg4:
  case Intrinsic::riscv_vsoxseg5:
  case Intrinsic::riscv_vsoxseg6:
  case Intrinsic::riscv_vsoxseg7:
  case Intrinsic::riscv_vsoxseg8:
  case Intrinsic::riscv_vsuxseg2:
  case Intrinsic::riscv_vsuxseg3:
  case Intrinsic::riscv_vsuxseg4:
  case Intrinsic::riscv_vsuxseg5:
  case Intrinsic::riscv_vsuxseg6:
  case Intrinsic::riscv_vsuxseg7:
  case Intrinsic::riscv_vsuxseg8:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vssseg2_mask:
  case Intrinsic::riscv_vssseg3_mask:
  case Intrinsic::riscv_vssseg4_mask:
  case Intrinsic::riscv_vssseg5_mask:
  case Intrinsic::riscv_vssseg6_mask:
  case Intrinsic::riscv_vssseg7_mask:
  case Intrinsic::riscv_vssseg8_mask:
  case Intrinsic::riscv_vsoxseg2_mask:
  case Intrinsic::riscv_vsoxseg3_mask:
  case Intrinsic::riscv_vsoxseg4_mask:
  case Intrinsic::riscv_vsoxseg5_mask:
  case Intrinsic::riscv_vsoxseg6_mask:
  case Intrinsic::riscv_vsoxseg7_mask:
  case Intrinsic::riscv_vsoxseg8_mask:
  case Intrinsic::riscv_vsuxseg2_mask:
  case Intrinsic::riscv_vsuxseg3_mask:
  case Intrinsic::riscv_vsuxseg4_mask:
  case Intrinsic::riscv_vsuxseg5_mask:
  case Intrinsic::riscv_vsuxseg6_mask:
  case Intrinsic::riscv_vsuxseg7_mask:
  case Intrinsic::riscv_vsuxseg8_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_sf_vlte8:
  case Intrinsic::riscv_sf_vlte16:
  case Intrinsic::riscv_sf_vlte32:
  case Intrinsic::riscv_sf_vlte64:
    Info.opc = ISD::INTRINSIC_VOID;
    Info.ptrVal = I.getArgOperand(1);
    switch (Intrinsic) {
    case Intrinsic::riscv_sf_vlte8:
      Info.memVT = MVT::i8;
      Info.align = Align(1);
      break;
    case Intrinsic::riscv_sf_vlte16:
      Info.memVT = MVT::i16;
      Info.align = Align(2);
      break;
    case Intrinsic::riscv_sf_vlte32:
      Info.memVT = MVT::i32;
      Info.align = Align(4);
      break;
    case Intrinsic::riscv_sf_vlte64:
      Info.memVT = MVT::i64;
      Info.align = Align(8);
      break;
    }
    Info.size = MemoryLocation::UnknownSize;
    Info.flags |= MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  case Intrinsic::riscv_sf_vste8:
  case Intrinsic::riscv_sf_vste16:
  case Intrinsic::riscv_sf_vste32:
  case Intrinsic::riscv_sf_vste64:
    Info.opc = ISD::INTRINSIC_VOID;
    Info.ptrVal = I.getArgOperand(1);
    switch (Intrinsic) {
    case Intrinsic::riscv_sf_vste8:
      Info.memVT = MVT::i8;
      Info.align = Align(1);
      break;
    case Intrinsic::riscv_sf_vste16:
      Info.memVT = MVT::i16;
      Info.align = Align(2);
      break;
    case Intrinsic::riscv_sf_vste32:
      Info.memVT = MVT::i32;
      Info.align = Align(4);
      break;
    case Intrinsic::riscv_sf_vste64:
      Info.memVT = MVT::i64;
      Info.align = Align(8);
      break;
    }
    Info.size = MemoryLocation::UnknownSize;
    Info.flags |= MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
}
 
bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                                const AddrMode &AM, Type *Ty,
                                                unsigned AS,
                                                Instruction *I) const {
  // No global is ever allowed as a base.
  if (AM.BaseGV)
    return false;
 
  // None of our addressing modes allows a scalable offset
  if (AM.ScalableOffset)
    return false;
 
  // RVV instructions only support register addressing.
  if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
    return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
 
  // The Xqcilo extension provides load/store instructions with a 26-bit signed
  // offset.
  if (Subtarget.hasVendorXqcilo()) {
    if (!isInt<26>(AM.BaseOffs))
      return false;
  } else if (!isInt<12>(AM.BaseOffs)) {
    // Otherwise require a 12-bit signed offset.
    return false;
  }
 
  switch (AM.Scale) {
  case 0: // "r+i" or just "i", depending on HasBaseReg.
    break;
  case 1:
    if (!AM.HasBaseReg) // allow "r+i".
      break;
    return false; // disallow "r+r" or "r+r+i".
  default:
    return false;
  }
 
  return true;
}
 
bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
  return isInt<12>(Imm);
}
 
bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
  // The Xqcilia extension provides add-immediate instructions with a 26-bit
  // signed immediate.
  if (Subtarget.hasVendorXqcilia())
    return isInt<26>(Imm);
  return isInt<12>(Imm);
}
 
// On RV32, 64-bit integers are split into their high and low parts and held
// in two different registers, so the trunc is free since the low register can
// just be used.
// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
// isTruncateFree?
bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
  if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
    return false;
  unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
  unsigned DestBits = DstTy->getPrimitiveSizeInBits();
  return (SrcBits == 64 && DestBits == 32);
}
 
bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
  // We consider i64->i32 free on RV64 since we have good selection of W
  // instructions that make promoting operations back to i64 free in many cases.
  if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
      !DstVT.isInteger())
    return false;
  unsigned SrcBits = SrcVT.getSizeInBits();
  unsigned DestBits = DstVT.getSizeInBits();
  return (SrcBits == 64 && DestBits == 32);
}
 
bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
  EVT SrcVT = Val.getValueType();
  // free truncate from vnsrl and vnsra
  if (Subtarget.hasVInstructions() &&
      (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
      SrcVT.isVector() && VT2.isVector()) {
    unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
    unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
    if (SrcBits == DestBits * 2) {
      return true;
    }
  }
  return TargetLowering::isTruncateFree(Val, VT2);
}
 
bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
  // Zexts are free if they can be combined with a load.
  // Don't advertise i32->i64 zextload as being free for RV64. It interacts
  // poorly with type legalization of compares preferring sext.
  if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
    EVT MemVT = LD->getMemoryVT();
    if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
        (LD->getExtensionType() == ISD::NON_EXTLOAD ||
         LD->getExtensionType() == ISD::ZEXTLOAD))
      return true;
  }
 
  return TargetLowering::isZExtFree(Val, VT2);
}
 
bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
  return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
}
 
bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
  return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
}
 
bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
  return Subtarget.hasCTZLike();
}
 
bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
  return Subtarget.hasCLZLike();
}
 
bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
    const Instruction &AndI) const {
  // We expect to be able to match a bit extraction instruction if the Zbs
  // extension is supported and the mask is a power of two. However, we
  // conservatively return false if the mask would fit in an ANDI instruction,
  // on the basis that it's possible the sinking+duplication of the AND in
  // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
  // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
  if (!Subtarget.hasBEXTILike())
    return false;
  ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
  if (!Mask)
    return false;
  return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
}
 
bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
  EVT VT = Y.getValueType();
 
  if (VT.isVector())
    return false;
 
  return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
         (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
}
 
bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
  EVT VT = Y.getValueType();
 
  if (!VT.isVector())
    return hasAndNotCompare(Y);
 
  // vmandn.mm
  if (VT.getVectorElementType() == MVT::i1)
    return Subtarget.hasVInstructions();
 
  return Subtarget.hasStdExtZvkb();
}
 
bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
  // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
  if (Subtarget.hasStdExtZbs())
    return X.getValueType().isScalarInteger();
  auto *C = dyn_cast<ConstantSDNode>(Y);
  // XTheadBs provides th.tst (similar to bexti), if Y is a constant
  if (Subtarget.hasVendorXTHeadBs())
    return C != nullptr;
  // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
  return C && C->getAPIntValue().ule(10);
}
 
bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
    unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
    SDValue Y) const {
  if (SelectOpcode != ISD::VSELECT)
    return false;
 
  // Only enable for rvv.
  if (!VT.isVector() || !Subtarget.hasVInstructions())
    return false;
 
  if (VT.isFixedLengthVector() && !isTypeLegal(VT))
    return false;
 
  return true;
}
 
bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                                            Type *Ty) const {
  assert(Ty->isIntegerTy());
 
  unsigned BitSize = Ty->getIntegerBitWidth();
  if (BitSize > Subtarget.getXLen())
    return false;
 
  // Fast path, assume 32-bit immediates are cheap.
  int64_t Val = Imm.getSExtValue();
  if (isInt<32>(Val))
    return true;
 
  // A constant pool entry may be more aligned than the load we're trying to
  // replace. If we don't support unaligned scalar mem, prefer the constant
  // pool.
  // TODO: Can the caller pass down the alignment?
  if (!Subtarget.enableUnalignedScalarMem())
    return true;
 
  // Prefer to keep the load if it would require many instructions.
  // This uses the same threshold we use for constant pools but doesn't
  // check useConstantPoolForLargeInts.
  // TODO: Should we keep the load only when we're definitely going to emit a
  // constant pool?
 
  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget);
  return Seq.size() <= Subtarget.getMaxBuildIntsCost();
}
 
bool RISCVTargetLowering::
    shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
        SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
        unsigned OldShiftOpcode, unsigned NewShiftOpcode,
        SelectionDAG &DAG) const {
  // One interesting pattern that we'd want to form is 'bit extract':
  //   ((1 >> Y) & 1) ==/!= 0
  // But we also need to be careful not to try to reverse that fold.
 
  // Is this '((1 >> Y) & 1)'?
  if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
    return false; // Keep the 'bit extract' pattern.
 
  // Will this be '((1 >> Y) & 1)' after the transform?
  if (NewShiftOpcode == ISD::SRL && CC->isOne())
    return true; // Do form the 'bit extract' pattern.
 
  // If 'X' is a constant, and we transform, then we will immediately
  // try to undo the fold, thus causing endless combine loop.
  // So only do the transform if X is not a constant. This matches the default
  // implementation of this function.
  return !XC;
}
 
bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
  unsigned Opc = VecOp.getOpcode();
 
  // Assume target opcodes can't be scalarized.
  // TODO - do we have any exceptions?
  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
    return false;
 
  // If the vector op is not supported, try to convert to scalar.
  EVT VecVT = VecOp.getValueType();
  if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
    return true;
 
  // If the vector op is supported, but the scalar op is not, the transform may
  // not be worthwhile.
  // Permit a vector binary operation can be converted to scalar binary
  // operation which is custom lowered with illegal type.
  EVT ScalarVT = VecVT.getScalarType();
  return isOperationLegalOrCustomOrPromote(Opc, ScalarVT) ||
         isOperationCustom(Opc, ScalarVT);
}
 
bool RISCVTargetLowering::isOffsetFoldingLegal(
    const GlobalAddressSDNode *GA) const {
  // In order to maximise the opportunity for common subexpression elimination,
  // keep a separate ADD node for the global address offset instead of folding
  // it in the global address node. Later peephole optimisations may choose to
  // fold it back in when profitable.
  return false;
}
 
// Returns 0-31 if the fli instruction is available for the type and this is
// legal FP immediate for the type. Returns -1 otherwise.
int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
  if (!Subtarget.hasStdExtZfa())
    return -1;
 
  bool IsSupportedVT = false;
  if (VT == MVT::f16) {
    IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
  } else if (VT == MVT::f32) {
    IsSupportedVT = true;
  } else if (VT == MVT::f64) {
    assert(Subtarget.hasStdExtD() && "Expect D extension");
    IsSupportedVT = true;
  }
 
  if (!IsSupportedVT)
    return -1;
 
  return RISCVLoadFPImm::getLoadFPImm(Imm);
}
 
bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                       bool ForCodeSize) const {
  bool IsLegalVT = false;
  if (VT == MVT::f16)
    IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
  else if (VT == MVT::f32)
    IsLegalVT = Subtarget.hasStdExtFOrZfinx();
  else if (VT == MVT::f64)
    IsLegalVT = Subtarget.hasStdExtDOrZdinx();
  else if (VT == MVT::bf16)
    IsLegalVT = Subtarget.hasStdExtZfbfmin();
 
  if (!IsLegalVT)
    return false;
 
  if (getLegalZfaFPImm(Imm, VT) >= 0)
    return true;
 
  // Some constants can be produced by fli+fneg.
  if (Imm.isNegative() && getLegalZfaFPImm(-Imm, VT) >= 0)
    return true;
 
  // Cannot create a 64 bit floating-point immediate value for rv32.
  if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
    // td can handle +0.0 or -0.0 already.
    // -0.0 can be created by fmv + fneg.
    return Imm.isZero();
  }
 
  // Special case: fmv + fneg
  if (Imm.isNegZero())
    return true;
 
  // Building an integer and then converting requires a fmv at the end of
  // the integer sequence. The fmv is not required for Zfinx.
  const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
  const int Cost =
      FmvCost + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(),
                                           Subtarget.getXLen(), Subtarget);
  return Cost <= FPImmCost;
}
 
// TODO: This is very conservative.
TargetLowering::ExtractSubvectorCost
RISCVTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
                                             unsigned Index) const {
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
      (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) {
    if ((Index % ResVT.getVectorNumElements()) == 0)
      return ExtractSubvectorCost::Free;
    return ExtractSubvectorCost::Expensive;
  }
 
  if (!Subtarget.hasVInstructions())
    return ExtractSubvectorCost::Expensive;
 
  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
    return ExtractSubvectorCost::Expensive;
 
  // Extracts from index 0 are just subreg extracts.
  if (Index == 0)
    return ExtractSubvectorCost::Free;
 
  // Only support extracting a fixed from a fixed vector for now.
  if (ResVT.isScalableVector() || SrcVT.isScalableVector())
    return ExtractSubvectorCost::Expensive;
 
  EVT EltVT = ResVT.getVectorElementType();
  assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
 
  // The smallest type we can slide is i8.
  if (EltVT == MVT::i1)
    return ExtractSubvectorCost::Expensive;
 
  unsigned ResElts = ResVT.getVectorNumElements();
  unsigned SrcElts = SrcVT.getVectorNumElements();
 
  unsigned MinVLen = Subtarget.getRealMinVLen();
  unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
 
  // If we're extracting only data from the first VLEN bits of the source
  // then we can always do this with an m1 vslidedown.vx.  Restricting the
  // Index ensures we can use a vslidedown.vi.
  // TODO: We can generalize this when the exact VLEN is known.
  if (Index + ResElts <= MinVLMAX && Index < 31)
    return ExtractSubvectorCost::Free;
 
  // Convervatively only handle extracting half of a vector.
  // TODO: We can do arbitrary slidedowns, but for now only support extracting
  // the upper half of a vector until we have more test coverage.
  // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
  // a cheap extract.  However, this case is important in practice for
  // shuffled extracts of longer vectors.  How resolve?
  if ((ResElts * 2) == SrcElts && Index == ResElts)
    return ExtractSubvectorCost::Free;
  return ExtractSubvectorCost::Expensive;
}
 
MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                      CallingConv::ID CC,
                                                      EVT VT) const {
  // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
  // We might still end up using a GPR but that will be decided based on ABI.
  if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
      !Subtarget.hasStdExtZfhminOrZhinxmin())
    return MVT::f32;
 
  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
 
unsigned
RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
                                     std::optional<MVT> RegisterVT) const {
  // Pair inline assembly operand
  if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
      *RegisterVT == MVT::Untyped)
    return 1;
 
  return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
}
 
unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                           CallingConv::ID CC,
                                                           EVT VT) const {
  // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
  // We might still end up using a GPR but that will be decided based on ABI.
  if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
      !Subtarget.hasStdExtZfhminOrZhinxmin())
    return 1;
 
  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
 
// Changes the condition code and swaps operands if necessary, so the SetCC
// operation matches one of the comparisons supported directly by branches
// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
// with 1/-1.
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
                                    ISD::CondCode &CC, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
  // If this is a single bit test that can't be handled by ANDI, shift the
  // bit to be tested to the MSB and perform a signed compare with 0.
  if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
      LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
      isa<ConstantSDNode>(LHS.getOperand(1)) &&
      // XAndesPerf supports branch on test bit.
      !Subtarget.hasVendorXAndesPerf()) {
    uint64_t Mask = LHS.getConstantOperandVal(1);
    if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
      unsigned ShAmt = 0;
      if (isPowerOf2_64(Mask)) {
        CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
        ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
      } else {
        ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
      }
 
      LHS = LHS.getOperand(0);
      if (ShAmt != 0)
        LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
                          DAG.getConstant(ShAmt, DL, LHS.getValueType()));
      return;
    }
  }
 
  if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
    int64_t C = RHSC->getSExtValue();
    switch (CC) {
    default: break;
    case ISD::SETGT:
      // Convert X > -1 to X >= 0.
      if (C == -1) {
        RHS = DAG.getConstant(0, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
          C != INT64_MAX && isInt<5>(C + 1)) {
        // We have a conditional move instruction for SETGE but not SETGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
        RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(C + 1)) {
        // We have a branch immediate instruction for SETGE but not SETGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
        RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      break;
    case ISD::SETLT:
      // Convert X < 1 to 0 >= X.
      if (C == 1) {
        RHS = LHS;
        LHS = DAG.getConstant(0, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      break;
    case ISD::SETUGT:
      if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
          C != INT64_MAX && isUInt<5>(C + 1)) {
        // We have a conditional move instruction for SETUGE but not SETUGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
        RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETUGE;
        return;
      }
      if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(C + 1)) {
        // We have a branch immediate instruction for SETUGE but not SETUGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
        // immediate.
        RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETUGE;
        return;
      }
      break;
    }
  }
 
  switch (CC) {
  default:
    break;
  case ISD::SETGT:
  case ISD::SETLE:
  case ISD::SETUGT:
  case ISD::SETULE:
    CC = ISD::getSetCCSwappedOperands(CC);
    std::swap(LHS, RHS);
    break;
  }
}
 
RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
  if (VT.isRISCVVectorTuple()) {
    if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv1i8x8)
      return RISCVVType::LMUL_F8;
    if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv2i8x8)
      return RISCVVType::LMUL_F4;
    if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv4i8x8)
      return RISCVVType::LMUL_F2;
    if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv8i8x8)
      return RISCVVType::LMUL_1;
    if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv16i8x4)
      return RISCVVType::LMUL_2;
    if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
      return RISCVVType::LMUL_4;
    llvm_unreachable("Invalid vector tuple type LMUL.");
  }
 
  assert(VT.isScalableVector() && "Expecting a scalable vector type");
  unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
  if (VT.getVectorElementType() == MVT::i1)
    KnownSize *= 8;
 
  switch (KnownSize) {
  default:
    llvm_unreachable("Invalid LMUL.");
  case 8:
    return RISCVVType::LMUL_F8;
  case 16:
    return RISCVVType::LMUL_F4;
  case 32:
    return RISCVVType::LMUL_F2;
  case 64:
    return RISCVVType::LMUL_1;
  case 128:
    return RISCVVType::LMUL_2;
  case 256:
    return RISCVVType::LMUL_4;
  case 512:
    return RISCVVType::LMUL_8;
  }
}
 
unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
  switch (LMul) {
  default:
    llvm_unreachable("Invalid LMUL.");
  case RISCVVType::LMUL_F8:
  case RISCVVType::LMUL_F4:
  case RISCVVType::LMUL_F2:
  case RISCVVType::LMUL_1:
    return RISCV::VRRegClassID;
  case RISCVVType::LMUL_2:
    return RISCV::VRM2RegClassID;
  case RISCVVType::LMUL_4:
    return RISCV::VRM4RegClassID;
  case RISCVVType::LMUL_8:
    return RISCV::VRM8RegClassID;
  }
}
 
unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
  RISCVVType::VLMUL LMUL = getLMUL(VT);
  if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
      LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
    static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
                  "Unexpected subreg numbering");
    return RISCV::sub_vrm1_0 + Index;
  }
  if (LMUL == RISCVVType::LMUL_2) {
    static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
                  "Unexpected subreg numbering");
    return RISCV::sub_vrm2_0 + Index;
  }
  if (LMUL == RISCVVType::LMUL_4) {
    static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
                  "Unexpected subreg numbering");
    return RISCV::sub_vrm4_0 + Index;
  }
  llvm_unreachable("Invalid vector type.");
}
 
unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
  if (VT.isRISCVVectorTuple()) {
    unsigned NF = VT.getRISCVVectorTupleNumFields();
    unsigned RegsPerField =
        std::max(1U, (unsigned)VT.getSizeInBits().getKnownMinValue() /
                         (NF * RISCV::RVVBitsPerBlock));
    switch (RegsPerField) {
    case 1:
      if (NF == 2)
        return RISCV::VRN2M1RegClassID;
      if (NF == 3)
        return RISCV::VRN3M1RegClassID;
      if (NF == 4)
        return RISCV::VRN4M1RegClassID;
      if (NF == 5)
        return RISCV::VRN5M1RegClassID;
      if (NF == 6)
        return RISCV::VRN6M1RegClassID;
      if (NF == 7)
        return RISCV::VRN7M1RegClassID;
      if (NF == 8)
        return RISCV::VRN8M1RegClassID;
      break;
    case 2:
      if (NF == 2)
        return RISCV::VRN2M2RegClassID;
      if (NF == 3)
        return RISCV::VRN3M2RegClassID;
      if (NF == 4)
        return RISCV::VRN4M2RegClassID;
      break;
    case 4:
      assert(NF == 2);
      return RISCV::VRN2M4RegClassID;
    default:
      break;
    }
    llvm_unreachable("Invalid vector tuple type RegClass.");
  }
 
  if (VT.getVectorElementType() == MVT::i1)
    return RISCV::VRRegClassID;
  return getRegClassIDForLMUL(getLMUL(VT));
}
 
// Attempt to decompose a subvector insert/extract between VecVT and
// SubVecVT via subregister indices. Returns the subregister index that
// can perform the subvector insert/extract with the given element index, as
// well as the index corresponding to any leftover subvectors that must be
// further inserted/extracted within the register class for SubVecVT.
std::pair<unsigned, unsigned>
RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
    MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
    const RISCVRegisterInfo *TRI) {
  static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
                 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
                 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
                "Register classes not ordered");
  unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
  unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
 
  // If VecVT is a vector tuple type, either it's the tuple type with same
  // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
  if (VecVT.isRISCVVectorTuple()) {
    if (VecRegClassID == SubRegClassID)
      return {RISCV::NoSubRegister, 0};
 
    assert(SubVecVT.isScalableVector() &&
           "Only allow scalable vector subvector.");
    assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
           "Invalid vector tuple insert/extract for vector and subvector with "
           "different LMUL.");
    return {getSubregIndexByMVT(VecVT, InsertExtractIdx), 0};
  }
 
  // Try to compose a subregister index that takes us from the incoming
  // LMUL>1 register class down to the outgoing one. At each step we half
  // the LMUL:
  //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
  // Note that this is not guaranteed to find a subregister index, such as
  // when we are extracting from one VR type to another.
  unsigned SubRegIdx = RISCV::NoSubRegister;
  for (const unsigned RCID :
       {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
    if (VecRegClassID > RCID && SubRegClassID <= RCID) {
      VecVT = VecVT.getHalfNumVectorElementsVT();
      bool IsHi =
          InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
      SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
                                            getSubregIndexByMVT(VecVT, IsHi));
      if (IsHi)
        InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
    }
  return {SubRegIdx, InsertExtractIdx};
}
 
// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
// stores for those types.
bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
      (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
    return false;
 
  return !Subtarget.useRVVForFixedLengthVectors() ||
         VT.isFixedLengthVectorOf(MVT::i1);
}
 
bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
  if (!ScalarTy.isSimple())
    return false;
  switch (ScalarTy.getSimpleVT().SimpleTy) {
  case MVT::iPTR:
    return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
  case MVT::i8:
  case MVT::i16:
  case MVT::i32:
    return Subtarget.hasVInstructions();
  case MVT::i64:
    return Subtarget.hasVInstructionsI64();
  case MVT::f16:
    return Subtarget.hasVInstructionsF16Minimal();
  case MVT::bf16:
    return Subtarget.hasVInstructionsBF16Minimal();
  case MVT::f32:
    return Subtarget.hasVInstructionsF32();
  case MVT::f64:
    return Subtarget.hasVInstructionsF64();
  default:
    return false;
  }
}
 
 
unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
  return NumRepeatedDivisors;
}
 
static SDValue getVLOperand(SDValue Op) {
  assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
         "Unexpected opcode");
  bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
  unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
  const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
      RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
  if (!II)
    return SDValue();
  return Op.getOperand(II->VLOperand + 1 + HasChain);
}
 
static bool useRVVForFixedLengthVectorVT(MVT VT,
                                         const RISCVSubtarget &Subtarget) {
  assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
  if (!Subtarget.useRVVForFixedLengthVectors())
    return false;
 
  // We only support a set of vector types with a consistent maximum fixed size
  // across all supported vector element types to avoid legalization issues.
  // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
  // fixed-length vector type we support is 1024 bytes.
  if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
    return false;
 
  unsigned MinVLen = Subtarget.getRealMinVLen();
 
  MVT EltVT = VT.getVectorElementType();
 
  // Don't use RVV for vectors we cannot scalarize if required.
  switch (EltVT.SimpleTy) {
  // i1 is supported but has different rules.
  default:
    return false;
  case MVT::i1:
    // Masks can only use a single register.
    if (VT.getVectorNumElements() > MinVLen)
      return false;
    MinVLen /= 8;
    break;
  case MVT::i8:
  case MVT::i16:
  case MVT::i32:
    break;
  case MVT::i64:
    if (!Subtarget.hasVInstructionsI64())
      return false;
    break;
  case MVT::f16:
    if (!Subtarget.hasVInstructionsF16Minimal())
      return false;
    break;
  case MVT::bf16:
    if (!Subtarget.hasVInstructionsBF16Minimal())
      return false;
    break;
  case MVT::f32:
    if (!Subtarget.hasVInstructionsF32())
      return false;
    break;
  case MVT::f64:
    if (!Subtarget.hasVInstructionsF64())
      return false;
    break;
  }
 
  // Reject elements larger than ELEN.
  if (EltVT.getSizeInBits() > Subtarget.getELen())
    return false;
 
  unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
  // Don't use RVV for types that don't fit.
  if (LMul > 8)
    return false;
 
  // TODO: Perhaps an artificial restriction, but worth having whilst getting
  // the base fixed length RVV support in place.
  if (!VT.isPow2VectorType())
    return false;
 
  return true;
}
 
bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
  return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
}
 
// Return the largest legal scalable vector type that matches VT's element type.
static MVT getContainerForFixedLengthVector(MVT VT,
                                            const RISCVSubtarget &Subtarget) {
  // This may be called before legal types are setup.
  assert(((VT.isFixedLengthVector() &&
           Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
          useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
         "Expected legal fixed length vector!");
 
  unsigned MinVLen = Subtarget.getRealMinVLen();
  unsigned MaxELen = Subtarget.getELen();
 
  MVT EltVT = VT.getVectorElementType();
  switch (EltVT.SimpleTy) {
  default:
    llvm_unreachable("unexpected element type for RVV container");
  case MVT::i1:
  case MVT::i8:
  case MVT::i16:
  case MVT::i32:
  case MVT::i64:
  case MVT::bf16:
  case MVT::f16:
  case MVT::f32:
  case MVT::f64: {
    // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
    // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
    // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
    unsigned NumElts =
        (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
    NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
    assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
    return MVT::getScalableVectorVT(EltVT, NumElts);
  }
  }
}
 
MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
  return ::getContainerForFixedLengthVector(VT, getSubtarget());
}
 
// Grow V to consume an entire RVV register.
static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
                                       const RISCVSubtarget &Subtarget) {
  assert(VT.isScalableVector() &&
         "Expected to convert into a scalable vector!");
  assert(V.getValueType().isFixedLengthVector() &&
         "Expected a fixed length vector operand!");
  SDLoc DL(V);
  return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), V, 0);
}
 
// Shrink V so it's just big enough to maintain a VT's worth of data.
static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
                                         const RISCVSubtarget &Subtarget) {
  assert(VT.isFixedLengthVector() &&
         "Expected to convert into a fixed length vector!");
  assert(V.getValueType().isScalableVector() &&
         "Expected a scalable vector operand!");
  SDLoc DL(V);
  return DAG.getExtractSubvector(DL, VT, V, 0);
}
 
/// Return the type of the mask type suitable for masking the provided
/// vector type.  This is simply an i1 element type vector of the same
/// (possibly scalable) length.
static MVT getMaskTypeFor(MVT VecVT) {
  assert(VecVT.isVector());
  ElementCount EC = VecVT.getVectorElementCount();
  return MVT::getVectorVT(MVT::i1, EC);
}
 
/// Creates an all ones mask suitable for masking a vector of type VecTy with
/// vector length VL.  .
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
                              SelectionDAG &DAG) {
  MVT MaskVT = getMaskTypeFor(VecVT);
  return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
}
 
static std::pair<SDValue, SDValue>
getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
                        const RISCVSubtarget &Subtarget) {
  assert(VecVT.isScalableVector() && "Expecting a scalable vector");
  SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
  SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
  return {Mask, VL};
}
 
static std::pair<SDValue, SDValue>
getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
                SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
  assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
  SDValue VL = DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
  SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
  return {Mask, VL};
}
 
// Gets the two common "VL" operands: an all-ones mask and the vector length.
// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
// the vector type that the fixed-length vector is contained in. Otherwise if
// VecVT is scalable, then ContainerVT should be the same as VecVT.
static std::pair<SDValue, SDValue>
getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
                const RISCVSubtarget &Subtarget) {
  if (VecVT.isFixedLengthVector())
    return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
                           Subtarget);
  assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
  return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
}
 
SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
                                          SelectionDAG &DAG) const {
  assert(VecVT.isScalableVector() && "Expected scalable vector");
  return DAG.getElementCount(DL, Subtarget.getXLenVT(),
                             VecVT.getVectorElementCount());
}
 
std::pair<unsigned, unsigned>
RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
                                        const RISCVSubtarget &Subtarget) {
  assert(VecVT.isScalableVector() && "Expected scalable vector");
 
  unsigned EltSize = VecVT.getScalarSizeInBits();
  unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
 
  unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
  unsigned MaxVLMAX =
      RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
 
  unsigned VectorBitsMin = Subtarget.getRealMinVLen();
  unsigned MinVLMAX =
      RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
 
  return std::make_pair(MinVLMAX, MaxVLMAX);
}
 
// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
// of either is (currently) supported. This can get us into an infinite loop
// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
// as a ..., etc.
// Until either (or both) of these can reliably lower any node, reporting that
// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
// which is not desirable.
bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
    EVT VT, unsigned DefinedValues) const {
  return false;
}
 
InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
  // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
  // implementation-defined.
  if (!VT.isVector())
    return InstructionCost::getInvalid();
  unsigned DLenFactor = Subtarget.getDLenFactor();
  unsigned Cost;
  if (VT.isScalableVector()) {
    unsigned LMul;
    bool Fractional;
    std::tie(LMul, Fractional) =
        RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT));
    if (Fractional)
      Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
    else
      Cost = (LMul * DLenFactor);
  } else {
    Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
  }
  return Cost;
}
 
 
/// Return the cost of a vrgather.vv instruction for the type VT.  vrgather.vv
/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
/// be by default.  VRGatherCostModel reflects available options.  Note that
/// operand (index and possibly mask) are handled separately.
InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
  auto LMULCost = getLMULCost(VT);
  bool Log2CostModel =
      Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
  if (Log2CostModel && LMULCost.isValid()) {
    unsigned Log = Log2_64(LMULCost.getValue());
    if (Log > 0)
      return LMULCost * Log;
  }
  return LMULCost * LMULCost;
}
 
/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
/// or may track the vrgather.vv cost. It is implementation-dependent.
InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const {
  return getLMULCost(VT);
}
 
/// Return the cost of a vslidedown.vx or vslideup.vx instruction
/// for the type VT.  (This does not cover the vslide1up or vslide1down
/// variants.)  Slides may be linear in the number of vregs implied by LMUL,
/// or may track the vrgather.vv cost. It is implementation-dependent.
InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const {
  return getLMULCost(VT);
}
 
/// Return the cost of a vslidedown.vi or vslideup.vi instruction
/// for the type VT.  (This does not cover the vslide1up or vslide1down
/// variants.)  Slides may be linear in the number of vregs implied by LMUL,
/// or may track the vrgather.vv cost. It is implementation-dependent.
InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const {
  return getLMULCost(VT);
}
 
static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
  // bf16 conversions are always promoted to f32.
  if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
      Op.getValueType() == MVT::bf16) {
    bool IsStrict = Op->isStrictFPOpcode();
 
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Val = DAG.getNode(Op.getOpcode(), DL, {MVT::f32, MVT::Other},
                                {Op.getOperand(0), Op.getOperand(1)});
      return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
                         {Op.getValueType(), MVT::Other},
                         {Val.getValue(1), Val.getValue(0),
                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
    }
    return DAG.getNode(
        ISD::FP_ROUND, DL, Op.getValueType(),
        DAG.getNode(Op.getOpcode(), DL, MVT::f32, Op.getOperand(0)),
        DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
  }
 
  // Other operations are legal.
  return Op;
}
 
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  // RISC-V FP-to-int conversions saturate to the destination register size, but
  // don't produce 0 for nan. We can use a conversion instruction and fix the
  // nan case with a compare and a select.
  SDValue Src = Op.getOperand(0);
 
  MVT DstVT = Op.getSimpleValueType();
  EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
 
  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
 
  if (!DstVT.isVector()) {
    // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
    // the result.
    if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
        Src.getValueType() == MVT::bf16) {
      Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
    }
 
    unsigned Opc;
    if (SatVT == DstVT)
      Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
    else if (DstVT == MVT::i64 && SatVT == MVT::i32)
      Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
    else
      return SDValue();
    // FIXME: Support other SatVTs by clamping before or after the conversion.
 
    SDLoc DL(Op);
    SDValue FpToInt = DAG.getNode(
        Opc, DL, DstVT, Src,
        DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
 
    if (Opc == RISCVISD::FCVT_WU_RV64)
      FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
 
    SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
    return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
                           ISD::CondCode::SETUO);
  }
 
  // Vectors.
 
  MVT DstEltVT = DstVT.getVectorElementType();
  MVT SrcVT = Src.getSimpleValueType();
  MVT SrcEltVT = SrcVT.getVectorElementType();
  unsigned SrcEltSize = SrcEltVT.getSizeInBits();
  unsigned DstEltSize = DstEltVT.getSizeInBits();
 
  // Only handle saturating to the destination type.
  if (SatVT != DstEltVT)
    return SDValue();
 
  MVT DstContainerVT = DstVT;
  MVT SrcContainerVT = SrcVT;
  if (DstVT.isFixedLengthVector()) {
    DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    assert(DstContainerVT.getVectorElementCount() ==
               SrcContainerVT.getVectorElementCount() &&
           "Expected same element count");
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  SDLoc DL(Op);
 
  auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
 
  SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                              {Src, Src, DAG.getCondCode(ISD::SETNE),
                               DAG.getUNDEF(Mask.getValueType()), Mask, VL});
 
  // Need to widen by more than 1 step, promote the FP type, then do a widening
  // convert.
  if (DstEltSize > (2 * SrcEltSize)) {
    assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
    MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
    Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
  }
 
  MVT CvtContainerVT = DstContainerVT;
  MVT CvtEltVT = DstEltVT;
  if (SrcEltSize > (2 * DstEltSize)) {
    CvtEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
    CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
  }
 
  unsigned RVVOpc =
      IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
  SDValue Res = DAG.getNode(RVVOpc, DL, CvtContainerVT, Src, Mask, VL);
 
  while (CvtContainerVT != DstContainerVT) {
    CvtEltVT = MVT::getIntegerVT(CvtEltVT.getSizeInBits() / 2);
    CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
    // Rounding mode here is arbitrary since we aren't shifting out any bits.
    unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
                                : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
    Res = DAG.getNode(ClipOpc, DL, CvtContainerVT, Res, Mask, VL);
  }
 
  SDValue SplatZero = DAG.getNode(
      RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
      DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
  Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
                    Res, DAG.getUNDEF(DstContainerVT), VL);
 
  if (DstVT.isFixedLengthVector())
    Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
 
  return Res;
}
 
static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
 
  // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
  // bf16 conversions are always promoted to f32.
  if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
      SrcVal.getValueType() == MVT::bf16) {
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Ext =
          DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                      {Op.getOperand(0), SrcVal});
      return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
                         {Ext.getValue(1), Ext.getValue(0)});
    }
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                       DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
  }
 
  // Other operations are legal.
  return Op;
}
 
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
  switch (Opc) {
  case ISD::FROUNDEVEN:
  case ISD::STRICT_FROUNDEVEN:
    return RISCVFPRndMode::RNE;
  case ISD::FTRUNC:
  case ISD::STRICT_FTRUNC:
    return RISCVFPRndMode::RTZ;
  case ISD::FFLOOR:
  case ISD::STRICT_FFLOOR:
    return RISCVFPRndMode::RDN;
  case ISD::FCEIL:
  case ISD::STRICT_FCEIL:
    return RISCVFPRndMode::RUP;
  case ISD::FROUND:
  case ISD::LROUND:
  case ISD::LLROUND:
  case ISD::STRICT_FROUND:
  case ISD::STRICT_LROUND:
  case ISD::STRICT_LLROUND:
    return RISCVFPRndMode::RMM;
  case ISD::FRINT:
  case ISD::LRINT:
  case ISD::LLRINT:
  case ISD::STRICT_FRINT:
  case ISD::STRICT_LRINT:
  case ISD::STRICT_LLRINT:
    return RISCVFPRndMode::DYN;
  }
 
  return RISCVFPRndMode::Invalid;
}
 
// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
// the integer domain and back. Taking care to avoid converting values that are
// nan or already correct.
static SDValue
lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isVector() && "Unexpected type");
 
  SDLoc DL(Op);
 
  SDValue Src = Op.getOperand(0);
 
  // Freeze the source since we are increasing the number of uses.
  Src = DAG.getFreeze(Src);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  // We do the conversion on the absolute value and fix the sign at the end.
  SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
 
  // Determine the largest integer that can be represented exactly. This and
  // values larger than it don't have any fractional bits so don't need to
  // be converted.
  const fltSemantics &FltSem = ContainerVT.getFltSemantics();
  unsigned Precision = APFloat::semanticsPrecision(FltSem);
  APFloat MaxVal = APFloat(FltSem);
  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
  SDValue MaxValNode =
      DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
  SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
                                    DAG.getUNDEF(ContainerVT), MaxValNode, VL);
 
  // If abs(Src) was larger than MaxVal or nan, keep it.
  MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
  Mask =
      DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
                  {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
                   Mask, Mask, VL});
 
  // Truncate to integer and convert back to FP.
  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Truncated;
 
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode");
  case ISD::FRINT:
  case ISD::FCEIL:
  case ISD::FFLOOR:
  case ISD::FROUND:
  case ISD::FROUNDEVEN: {
    RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
    assert(FRM != RISCVFPRndMode::Invalid);
    Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
                            DAG.getTargetConstant(FRM, DL, XLenVT), VL);
    break;
  }
  case ISD::FTRUNC:
    Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
                            Mask, VL);
    break;
  case ISD::FNEARBYINT:
    Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
                            Mask, VL);
    break;
  }
 
  // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
  if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
    Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
                            Mask, VL);
 
  // Restore the original sign so that -0.0 is preserved.
  Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
                          Src, Src, Mask, VL);
 
  if (!VT.isFixedLengthVector())
    return Truncated;
 
  return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
}
 
// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
// qNan and converting the new source to integer and back to FP.
static SDValue
lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                            const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue Chain = Op.getOperand(0);
  SDValue Src = Op.getOperand(1);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  // Freeze the source since we are increasing the number of uses.
  Src = DAG.getFreeze(Src);
 
  // Convert sNan to qNan by executing x + x for all unordered element x in Src.
  MVT MaskVT = Mask.getSimpleValueType();
  SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
                                DAG.getVTList(MaskVT, MVT::Other),
                                {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
                                 DAG.getUNDEF(MaskVT), Mask, VL});
  Chain = Unorder.getValue(1);
  Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
                    DAG.getVTList(ContainerVT, MVT::Other),
                    {Chain, Src, Src, Src, Unorder, VL});
  Chain = Src.getValue(1);
 
  // We do the conversion on the absolute value and fix the sign at the end.
  SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
 
  // Determine the largest integer that can be represented exactly. This and
  // values larger than it don't have any fractional bits so don't need to
  // be converted.
  const fltSemantics &FltSem = ContainerVT.getFltSemantics();
  unsigned Precision = APFloat::semanticsPrecision(FltSem);
  APFloat MaxVal = APFloat(FltSem);
  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
  SDValue MaxValNode =
      DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
  SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
                                    DAG.getUNDEF(ContainerVT), MaxValNode, VL);
 
  // If abs(Src) was larger than MaxVal or nan, keep it.
  Mask = DAG.getNode(
      RISCVISD::SETCC_VL, DL, MaskVT,
      {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
 
  // Truncate to integer and convert back to FP.
  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Truncated;
 
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode");
  case ISD::STRICT_FCEIL:
  case ISD::STRICT_FFLOOR:
  case ISD::STRICT_FROUND:
  case ISD::STRICT_FROUNDEVEN: {
    RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
    assert(FRM != RISCVFPRndMode::Invalid);
    Truncated = DAG.getNode(
        RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
        {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
    break;
  }
  case ISD::STRICT_FTRUNC:
    Truncated =
        DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
                    DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
    break;
  case ISD::STRICT_FNEARBYINT:
    Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
                            DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
                            Mask, VL);
    break;
  }
  Chain = Truncated.getValue(1);
 
  // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
  if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
    Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
                            DAG.getVTList(ContainerVT, MVT::Other), Chain,
                            Truncated, Mask, VL);
    Chain = Truncated.getValue(1);
  }
 
  // Restore the original sign so that -0.0 is preserved.
  Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
                          Src, Src, Mask, VL);
 
  if (VT.isFixedLengthVector())
    Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
  return DAG.getMergeValues({Truncated, Chain}, DL);
}
 
static SDValue
lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  if (VT.isVector())
    return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
 
  if (DAG.shouldOptForSize())
    return SDValue();
 
  SDLoc DL(Op);
  SDValue Src = Op.getOperand(0);
 
  // Create an integer the size of the mantissa with the MSB set. This and all
  // values larger than it don't have any fractional bits so don't need to be
  // converted.
  const fltSemantics &FltSem = VT.getFltSemantics();
  unsigned Precision = APFloat::semanticsPrecision(FltSem);
  APFloat MaxVal = APFloat(FltSem);
  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
  SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
 
  RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
  return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
                     DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
}
 
// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
                                       const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT DstVT = Op.getSimpleValueType();
  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();
  assert(SrcVT.isVector() && DstVT.isVector() &&
         !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
         "Unexpected type");
 
  MVT DstContainerVT = DstVT;
  MVT SrcContainerVT = SrcVT;
 
  if (DstVT.isFixedLengthVector()) {
    DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
 
  // [b]f16 -> f32
  MVT SrcElemType = SrcVT.getVectorElementType();
  if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
    MVT F32VT = SrcContainerVT.changeVectorElementType(MVT::f32);
    Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, F32VT, Src, Mask, VL);
  }
 
  SDValue Res =
      DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
                  DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
                                        Subtarget.getXLenVT()),
                  VL);
 
  if (!DstVT.isFixedLengthVector())
    return Res;
 
  return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
}
 
static SDValue
getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
              const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
              SDValue Offset, SDValue Mask, SDValue VL,
              unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
  if (Passthru.isUndef())
    Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
  SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
  return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
}
 
static SDValue
getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
            EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
            SDValue VL,
            unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
  if (Passthru.isUndef())
    Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
  SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
  return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
}
 
struct VIDSequence {
  int64_t StepNumerator;
  unsigned StepDenominator;
  int64_t Addend;
};
 
static std::optional<APInt> getExactInteger(const APFloat &APF,
                                            uint32_t BitWidth) {
  // We will use a SINT_TO_FP to materialize this constant so we should use a
  // signed APSInt here.
  APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
  // We use an arbitrary rounding mode here. If a floating-point is an exact
  // integer (e.g., 1.0), the rounding mode does not affect the output value. If
  // the rounding mode changes the output value, then it is not an exact
  // integer.
  RoundingMode ArbitraryRM = RoundingMode::TowardZero;
  bool IsExact;
  // If it is out of signed integer range, it will return an invalid operation.
  // If it is not an exact integer, IsExact is false.
  if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
       APFloatBase::opInvalidOp) ||
      !IsExact)
    return std::nullopt;
  return ValInt.extractBits(BitWidth, 0);
}
 
// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
// to the (non-zero) step S and start value X. This can be then lowered as the
// RVV sequence (VID * S) + X, for example.
// The step S is represented as an integer numerator divided by a positive
// denominator. Note that the implementation currently only identifies
// sequences in which either the numerator is +/- 1 or the denominator is 1. It
// cannot detect 2/3, for example.
// Note that this method will also match potentially unappealing index
// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
// determine whether this is worth generating code for.
//
// EltSizeInBits is the size of the type that the sequence will be calculated
// in, i.e. SEW for build_vectors or XLEN for address calculations.
static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
                                                      unsigned EltSizeInBits) {
  assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
  if (!cast<BuildVectorSDNode>(Op)->isConstant())
    return std::nullopt;
  bool IsInteger = Op.getValueType().isInteger();
 
  std::optional<unsigned> SeqStepDenom;
  std::optional<APInt> SeqStepNum;
  std::optional<APInt> SeqAddend;
  std::optional<std::pair<APInt, unsigned>> PrevElt;
  assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
 
  // First extract the ops into a list of constant integer values. This may not
  // be possible for floats if they're not all representable as integers.
  SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
  const unsigned OpSize = Op.getScalarValueSizeInBits();
  for (auto [Idx, Elt] : enumerate(Op->op_values())) {
    if (Elt.isUndef()) {
      Elts[Idx] = std::nullopt;
      continue;
    }
    if (IsInteger) {
      Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
    } else {
      auto ExactInteger =
          getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
      if (!ExactInteger)
        return std::nullopt;
      Elts[Idx] = *ExactInteger;
    }
  }
 
  for (auto [Idx, Elt] : enumerate(Elts)) {
    // Assume undef elements match the sequence; we just have to be careful
    // when interpolating across them.
    if (!Elt)
      continue;
 
    if (PrevElt) {
      // Calculate the step since the last non-undef element, and ensure
      // it's consistent across the entire sequence.
      unsigned IdxDiff = Idx - PrevElt->second;
      APInt ValDiff = *Elt - PrevElt->first;
 
      // A zero-value value difference means that we're somewhere in the middle
      // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
      // step change before evaluating the sequence.
      if (ValDiff == 0)
        continue;
 
      int64_t Remainder = ValDiff.srem(IdxDiff);
      // Normalize the step if it's greater than 1.
      if (Remainder != ValDiff.getSExtValue()) {
        // The difference must cleanly divide the element span.
        if (Remainder != 0)
          return std::nullopt;
        ValDiff = ValDiff.sdiv(IdxDiff);
        IdxDiff = 1;
      }
 
      if (!SeqStepNum)
        SeqStepNum = ValDiff;
      else if (ValDiff != SeqStepNum)
        return std::nullopt;
 
      if (!SeqStepDenom)
        SeqStepDenom = IdxDiff;
      else if (IdxDiff != *SeqStepDenom)
        return std::nullopt;
    }
 
    // Record this non-undef element for later.
    if (!PrevElt || PrevElt->first != *Elt)
      PrevElt = std::make_pair(*Elt, Idx);
  }
 
  // We need to have logged a step for this to count as a legal index sequence.
  if (!SeqStepNum || !SeqStepDenom)
    return std::nullopt;
 
  // Loop back through the sequence and validate elements we might have skipped
  // while waiting for a valid step. While doing this, log any sequence addend.
  for (auto [Idx, Elt] : enumerate(Elts)) {
    if (!Elt)
      continue;
    APInt ExpectedVal =
        (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
         *SeqStepNum)
            .sdiv(*SeqStepDenom);
 
    APInt Addend = *Elt - ExpectedVal;
    if (!SeqAddend)
      SeqAddend = Addend;
    else if (Addend != SeqAddend)
      return std::nullopt;
  }
 
  assert(SeqAddend && "Must have an addend if we have a step");
 
  return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
                     SeqAddend->getSExtValue()};
}
 
// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
// and lower it as a VRGATHER_VX_VL from the source vector.
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
                                  SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return SDValue();
  SDValue Src = SplatVal.getOperand(0);
  // Don't perform this optimization for i1 vectors, or if the element types are
  // different
  // FIXME: Support i1 vectors, maybe by promoting to i8?
  MVT EltTy = VT.getVectorElementType();
  if (EltTy == MVT::i1 ||
      !DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
    return SDValue();
  MVT SrcVT = Src.getSimpleValueType();
  if (EltTy != SrcVT.getVectorElementType())
    return SDValue();
  SDValue Idx = SplatVal.getOperand(1);
  // The index must be a legal type.
  if (Idx.getValueType() != Subtarget.getXLenVT())
    return SDValue();
 
  // Check that we know Idx lies within VT
  if (!TypeSize::isKnownLE(SrcVT.getSizeInBits(), VT.getSizeInBits())) {
    auto *CIdx = dyn_cast<ConstantSDNode>(Idx);
    if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
      return SDValue();
  }
 
  // Convert fixed length vectors to scalable
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  MVT SrcContainerVT = SrcVT;
  if (SrcVT.isFixedLengthVector()) {
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  // Put Vec in a VT sized vector
  if (SrcContainerVT.getVectorMinNumElements() <
      ContainerVT.getVectorMinNumElements())
    Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), Src, 0);
  else
    Src = DAG.getExtractSubvector(DL, ContainerVT, Src, 0);
 
  // We checked that Idx fits inside VT earlier
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
  SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Src,
                               Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
  if (VT.isFixedLengthVector())
    Gather = convertFromScalableVector(VT, Gather, DAG, Subtarget);
  return Gather;
}
 
static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
    int64_t StepNumerator = SimpleVID->StepNumerator;
    unsigned StepDenominator = SimpleVID->StepDenominator;
    int64_t Addend = SimpleVID->Addend;
 
    assert(StepNumerator != 0 && "Invalid step");
    bool Negate = false;
    int64_t SplatStepVal = StepNumerator;
    unsigned StepOpcode = ISD::MUL;
    // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
    // anyway as the shift of 63 won't fit in uimm5.
    if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
        isPowerOf2_64(std::abs(StepNumerator))) {
      Negate = StepNumerator < 0;
      StepOpcode = ISD::SHL;
      SplatStepVal = Log2_64(std::abs(StepNumerator));
    }
 
    // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
    // since it's the immediate value many RVV instructions accept. There is
    // no vmul.vi instruction so ensure multiply constant can fit in a
    // single addi instruction.  For the addend, we allow up to 32 bits..
    if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
         (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
        isPowerOf2_32(StepDenominator) &&
        (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(Addend)) {
      MVT VIDVT =
          VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
      MVT VIDContainerVT = getContainerForFixedLengthVector(VIDVT, Subtarget);
      SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
      // Convert right out of the scalable type so we can use standard ISD
      // nodes for the rest of the computation. If we used scalable types with
      // these, we'd lose the fixed-length vector info and generate worse
      // vsetvli code.
      VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
      if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
          (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
        SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
        VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
      }
      if (StepDenominator != 1) {
        SDValue SplatStep =
            DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
        VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
      }
      if (Addend != 0 || Negate) {
        SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
        VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
                          VID);
      }
      if (VT.isFloatingPoint()) {
        // TODO: Use vfwcvt to reduce register pressure.
        VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
      }
      return VID;
    }
  }
 
  return SDValue();
}
 
/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
/// which constitute a large proportion of the elements. In such cases we can
/// splat a vector with the dominant element and make up the shortfall with
/// INSERT_VECTOR_ELTs.  Returns SDValue if not profitable.
/// Note that this includes vectors of 2 elements by association. The
/// upper-most element is the "dominant" one, allowing us to use a splat to
/// "insert" the upper element, and an insert of the lower element at position
/// 0, which improves codegen.
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
                                                 const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned NumElts = Op.getNumOperands();
 
  SDValue DominantValue;
  unsigned MostCommonCount = 0;
  DenseMap<SDValue, unsigned> ValueCounts;
  unsigned NumUndefElts =
      count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
 
  // Track the number of scalar loads we know we'd be inserting, estimated as
  // any non-zero floating-point constant. Other kinds of element are either
  // already in registers or are materialized on demand. The threshold at which
  // a vector load is more desirable than several scalar materializion and
  // vector-insertion instructions is not known.
  unsigned NumScalarLoads = 0;
 
  for (SDValue V : Op->op_values()) {
    if (V.isUndef())
      continue;
 
    unsigned &Count = ValueCounts[V];
    if (0 == Count)
      if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
        NumScalarLoads += !CFP->isPosZero();
 
    // Is this value dominant? In case of a tie, prefer the highest element as
    // it's cheaper to insert near the beginning of a vector than it is at the
    // end.
    if (++Count >= MostCommonCount) {
      DominantValue = V;
      MostCommonCount = Count;
    }
  }
 
  assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
  unsigned NumDefElts = NumElts - NumUndefElts;
  unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
 
  // Don't perform this optimization when optimizing for size, since
  // materializing elements and inserting them tends to cause code bloat.
  if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
      (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
      ((MostCommonCount > DominantValueCountThreshold) ||
       (ValueCounts.size() <= Log2_32(NumDefElts)))) {
    // Start by splatting the most common element.
    SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
 
    DenseSet<SDValue> Processed{DominantValue};
 
    // We can handle an insert into the last element (of a splat) via
    // v(f)slide1down.  This is slightly better than the vslideup insert
    // lowering as it avoids the need for a vector group temporary.  It
    // is also better than using vmerge.vx as it avoids the need to
    // materialize the mask in a vector register.
    if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
        !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
        LastOp != DominantValue) {
      Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
      auto OpCode =
        VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
      if (!VT.isFloatingPoint())
        LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
      Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
                        LastOp, Mask, VL);
      Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
      Processed.insert(LastOp);
    }
 
    MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
    for (const auto &OpIdx : enumerate(Op->ops())) {
      const SDValue &V = OpIdx.value();
      if (V.isUndef() || !Processed.insert(V).second)
        continue;
      if (ValueCounts[V] == 1) {
        Vec = DAG.getInsertVectorElt(DL, Vec, V, OpIdx.index());
      } else {
        // Blend in all instances of this value using a VSELECT, using a
        // mask where each bit signals whether that element is the one
        // we're after.
        SmallVector<SDValue> Ops;
        transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
          return DAG.getConstant(V == V1, DL, XLenVT);
        });
        Vec = DAG.getNode(ISD::VSELECT, DL, VT,
                          DAG.getBuildVector(SelMaskTy, DL, Ops),
                          DAG.getSplatBuildVector(VT, DL, V), Vec);
      }
    }
 
    return Vec;
  }
 
  return SDValue();
}
 
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned NumElts = Op.getNumOperands();
 
  if (VT.getVectorElementType() == MVT::i1) {
    if (ISD::isBuildVectorAllZeros(Op.getNode())) {
      SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
      return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
    }
 
    if (ISD::isBuildVectorAllOnes(Op.getNode())) {
      SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
      return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
    }
 
    // Lower constant mask BUILD_VECTORs via an integer vector type, in
    // scalar integer chunks whose bit-width depends on the number of mask
    // bits and XLEN.
    // First, determine the most appropriate scalar integer type to use. This
    // is at most XLenVT, but may be shrunk to a smaller vector element type
    // according to the size of the final vector - use i8 chunks rather than
    // XLenVT if we're producing a v8i1. This results in more consistent
    // codegen across RV32 and RV64.
    unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
    NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
    // If we have to use more than one INSERT_VECTOR_ELT then this
    // optimization is likely to increase code size; avoid performing it in
    // such a case. We can use a load from a constant pool in this case.
    if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
      return SDValue();
    // Now we can create our integer vector type. Note that it may be larger
    // than the resulting mask type: v4i1 would use v1i8 as its integer type.
    unsigned IntegerViaVecElts = divideCeil(NumElts, NumViaIntegerBits);
    MVT IntegerViaVecVT =
      MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
                       IntegerViaVecElts);
 
    uint64_t Bits = 0;
    unsigned BitPos = 0, IntegerEltIdx = 0;
    SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
 
    for (unsigned I = 0; I < NumElts;) {
      SDValue V = Op.getOperand(I);
      bool BitValue = !V.isUndef() && V->getAsZExtVal();
      Bits |= ((uint64_t)BitValue << BitPos);
      ++BitPos;
      ++I;
 
      // Once we accumulate enough bits to fill our scalar type or process the
      // last element, insert into our vector and clear our accumulated data.
      if (I % NumViaIntegerBits == 0 || I == NumElts) {
        if (NumViaIntegerBits <= 32)
          Bits = SignExtend64<32>(Bits);
        SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
        Elts[IntegerEltIdx] = Elt;
        Bits = 0;
        BitPos = 0;
        IntegerEltIdx++;
      }
    }
 
    SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
 
    if (NumElts < NumViaIntegerBits) {
      // If we're producing a smaller vector than our minimum legal integer
      // type, bitcast to the equivalent (known-legal) mask type, and extract
      // our final mask.
      assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
      Vec = DAG.getBitcast(MVT::v8i1, Vec);
      Vec = DAG.getExtractSubvector(DL, VT, Vec, 0);
    } else {
      // Else we must have produced an integer type with the same size as the
      // mask type; bitcast for the final result.
      assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
      Vec = DAG.getBitcast(VT, Vec);
    }
 
    return Vec;
  }
 
  if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
    unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
                                        : RISCVISD::VMV_V_X_VL;
    if (!VT.isFloatingPoint())
      Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
    Splat =
        DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
    return convertFromScalableVector(VT, Splat, DAG, Subtarget);
  }
 
  // Try and match index sequences, which we can lower to the vid instruction
  // with optional modifications. An all-undef vector is matched by
  // getSplatValue, above.
  if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
    return Res;
 
  // For very small build_vectors, use a single scalar insert of a constant.
  // TODO: Base this on constant rematerialization cost, not size.
  const unsigned EltBitSize = VT.getScalarSizeInBits();
  if (VT.getSizeInBits() <= 32 &&
      ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
    MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
    assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
           "Unexpected sequence type");
    // If we can use the original VL with the modified element type, this
    // means we only have a VTYPE toggle, not a VL toggle.  TODO: Should this
    // be moved into InsertVSETVLI?
    unsigned ViaVecLen =
      (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
    MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
 
    uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
    uint64_t SplatValue = 0;
    // Construct the amalgamated value at this larger vector type.
    for (const auto &OpIdx : enumerate(Op->op_values())) {
      const auto &SeqV = OpIdx.value();
      if (!SeqV.isUndef())
        SplatValue |=
            ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
    }
 
    // On RV64, sign-extend from 32 to 64 bits where possible in order to
    // achieve better constant materializion.
    // On RV32, we need to sign-extend to use getSignedConstant.
    if (ViaIntVT == MVT::i32)
      SplatValue = SignExtend64<32>(SplatValue);
 
    SDValue Vec = DAG.getInsertVectorElt(
        DL, DAG.getUNDEF(ViaVecVT),
        DAG.getSignedConstant(SplatValue, DL, XLenVT), 0);
    if (ViaVecLen != 1)
      Vec = DAG.getExtractSubvector(DL, MVT::getVectorVT(ViaIntVT, 1), Vec, 0);
    return DAG.getBitcast(VT, Vec);
  }
 
 
  // Attempt to detect "hidden" splats, which only reveal themselves as splats
  // when re-interpreted as a vector with a larger element type. For example,
  //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
  // could be instead splat as
  //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
  // TODO: This optimization could also work on non-constant splats, but it
  // would require bit-manipulation instructions to construct the splat value.
  SmallVector<SDValue> Sequence;
  const auto *BV = cast<BuildVectorSDNode>(Op);
  if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
      ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
      BV->getRepeatedSequence(Sequence) &&
      (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
    unsigned SeqLen = Sequence.size();
    MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
    assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
            ViaIntVT == MVT::i64) &&
           "Unexpected sequence type");
 
    // If we can use the original VL with the modified element type, this
    // means we only have a VTYPE toggle, not a VL toggle.  TODO: Should this
    // be moved into InsertVSETVLI?
    const unsigned RequiredVL = NumElts / SeqLen;
    const unsigned ViaVecLen =
      (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
      NumElts : RequiredVL;
    MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
 
    unsigned EltIdx = 0;
    uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
    uint64_t SplatValue = 0;
    // Construct the amalgamated value which can be splatted as this larger
    // vector type.
    for (const auto &SeqV : Sequence) {
      if (!SeqV.isUndef())
        SplatValue |=
            ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
      EltIdx++;
    }
 
    // On RV64, sign-extend from 32 to 64 bits where possible in order to
    // achieve better constant materializion.
    // On RV32, we need to sign-extend to use getSignedConstant.
    if (ViaIntVT == MVT::i32)
      SplatValue = SignExtend64<32>(SplatValue);
 
    // Since we can't introduce illegal i64 types at this stage, we can only
    // perform an i64 splat on RV32 if it is its own sign-extended value. That
    // way we can use RVV instructions to splat.
    assert((ViaIntVT.bitsLE(XLenVT) ||
            (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
           "Unexpected bitcast sequence");
    if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
      SDValue ViaVL =
          DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
      MVT ViaContainerVT =
          getContainerForFixedLengthVector(ViaVecVT, Subtarget);
      SDValue Splat =
          DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
                      DAG.getUNDEF(ViaContainerVT),
                      DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
      Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
      if (ViaVecLen != RequiredVL)
        Splat = DAG.getExtractSubvector(
            DL, MVT::getVectorVT(ViaIntVT, RequiredVL), Splat, 0);
      return DAG.getBitcast(VT, Splat);
    }
  }
 
  // If the number of signbits allows, see if we can lower as a <N x i8>.
  // Our main goal here is to reduce LMUL (and thus work) required to
  // build the constant, but we will also narrow if the resulting
  // narrow vector is known to materialize cheaply.
  // TODO: We really should be costing the smaller vector.  There are
  // profitable cases this misses.
  if (EltBitSize > 8 && VT.isInteger() &&
      (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
      DAG.ComputeMaxSignificantBits(Op) <= 8) {
    SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
                                        DL, Op->ops());
    Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
                                     Source, DAG, Subtarget);
    SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
    return convertFromScalableVector(VT, Res, DAG, Subtarget);
  }
 
  if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
    return Res;
 
  // For constant vectors, use generic constant pool lowering.  Otherwise,
  // we'd have to materialize constants in GPRs just to move them into the
  // vector.
  return SDValue();
}
 
static unsigned getPACKOpcode(unsigned DestBW,
                              const RISCVSubtarget &Subtarget) {
  switch (DestBW) {
  default:
    llvm_unreachable("Unsupported pack size");
  case 16:
    return RISCV::PACKH;
  case 32:
    return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
  case 64:
    assert(Subtarget.is64Bit());
    return RISCV::PACK;
  }
}
 
/// Double the element size of the build vector to reduce the number
/// of vslide1down in the build vector chain.  In the worst case, this
/// trades three scalar operations for 1 vector operation.  Scalar
/// operations are generally lower latency, and for out-of-order cores
/// we also benefit from additional parallelism.
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
                                          const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
  MVT ElemVT = VT.getVectorElementType();
  if (!ElemVT.isInteger())
    return SDValue();
 
  // TODO: Relax these architectural restrictions, possibly with costing
  // of the actual instructions required.
  if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
    return SDValue();
 
  unsigned NumElts = VT.getVectorNumElements();
  unsigned ElemSizeInBits = ElemVT.getSizeInBits();
  if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) ||
      NumElts % 2 != 0)
    return SDValue();
 
  // Produce [B,A] packed into a type twice as wide.  Note that all
  // scalars are XLenVT, possibly masked (see below).
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Mask = DAG.getConstant(
      APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
  auto pack = [&](SDValue A, SDValue B) {
    // Bias the scheduling of the inserted operations to near the
    // definition of the element - this tends to reduce register
    // pressure overall.
    SDLoc ElemDL(B);
    if (Subtarget.hasStdExtZbkb())
      // Note that we're relying on the high bits of the result being
      // don't care.  For PACKW, the result is *sign* extended.
      return SDValue(
          DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
                             ElemDL, XLenVT, A, B),
          0);
 
    A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
    B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
    SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
    return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
                       DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt),
                       SDNodeFlags::Disjoint);
  };
 
  SmallVector<SDValue> NewOperands;
  NewOperands.reserve(NumElts / 2);
  for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
    NewOperands.push_back(pack(Op.getOperand(i), Op.getOperand(i + 1)));
  assert(NumElts == NewOperands.size() * 2);
  MVT WideVT = MVT::getIntegerVT(ElemSizeInBits * 2);
  MVT WideVecVT = MVT::getVectorVT(WideVT, NumElts / 2);
  return DAG.getNode(ISD::BITCAST, DL, VT,
                     DAG.getBuildVector(WideVecVT, DL, NewOperands));
}
 
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT EltVT = VT.getVectorElementType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  SDLoc DL(Op);
 
  if (!Subtarget.is64Bit() && Subtarget.hasStdExtP()) {
    if (VT == MVT::v2i16) {
      SDValue Lo = DAG.getBitcast(
          MVT::v2i16,
          DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32));
      SDValue Hi = DAG.getBitcast(
          MVT::v2i16,
          DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32));
      return DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16, Lo, Hi);
    }
 
    if (VT == MVT::v4i8) {
      // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b,
      // d))
      SDValue Val0 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(0));
      SDValue Val1 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(1));
      SDValue Val2 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(2));
      SDValue Val3 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(3));
      SDValue Concat1 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val0, Val2);
      SDValue Concat2 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val1, Val3);
      SDValue PPairE =
          DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v8i8, Concat1, Concat2);
 
      SDValue Lo = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 0);
      SDValue Hi = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 4);
 
      return DAG.getBitcast(MVT::v4i8,
                            DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16,
                                        DAG.getBitcast(MVT::v2i16, Lo),
                                        DAG.getBitcast(MVT::v2i16, Hi)));
    }
 
    llvm_unreachable("Unexpected RV32 P BUILD_VECTOR type");
  }
 
  // Proper support for f16 requires Zvfh. bf16 always requires special
  // handling. We need to cast the scalar to integer and create an integer
  // build_vector.
  if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
      (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
    MVT IVT = VT.changeVectorElementType(MVT::i16);
    SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
    for (const auto &[I, U] : enumerate(Op->ops())) {
      SDValue Elem = U.get();
      if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
          (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
        // Called by LegalizeDAG, we need to use XLenVT operations since we
        // can't create illegal types.
        if (auto *C = dyn_cast<ConstantFPSDNode>(Elem)) {
          // Manually constant fold so the integer build_vector can be lowered
          // better. Waiting for DAGCombine will be too late.
          APInt V =
              C->getValueAPF().bitcastToAPInt().sext(XLenVT.getSizeInBits());
          NewOps[I] = DAG.getConstant(V, DL, XLenVT);
        } else {
          NewOps[I] = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Elem);
        }
      } else {
        // Called by scalar type legalizer, we can use i16.
        NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
      }
    }
    SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, DL, IVT, NewOps);
    return DAG.getBitcast(VT, Res);
  }
 
  if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
      ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
    return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  if (VT.getVectorElementType() == MVT::i1) {
    // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
    // vector type, we have a legal equivalently-sized i8 type, so we can use
    // that.
    MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
    SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
 
    SDValue WideVec;
    if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
      // For a splat, perform a scalar truncate before creating the wider
      // vector.
      Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
                          DAG.getConstant(1, DL, Splat.getValueType()));
      WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
    } else {
      SmallVector<SDValue, 8> Ops(Op->op_values());
      WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
      SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
      WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
    }
 
    return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
  }
 
  if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
    if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
      return Gather;
 
    if (!VT.isFloatingPoint())
      Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
 
    // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
    // pressure at high LMUL.
    bool IsScalar = all_of(Op->ops().drop_front(),
                           [](const SDUse &U) { return U.get().isUndef(); });
    unsigned Opc =
        VT.isFloatingPoint()
            ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
            : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
    Splat =
        DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
    return convertFromScalableVector(VT, Splat, DAG, Subtarget);
  }
 
  if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
    return Res;
 
  // If we're compiling for an exact VLEN value, we can split our work per
  // register in the register group.
  if (const auto VLen = Subtarget.getRealVLen();
      VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
    MVT ElemVT = VT.getVectorElementType();
    unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
    EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
    MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
    assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
 
    // The following semantically builds up a fixed length concat_vector
    // of the component build_vectors.  We eagerly lower to scalable and
    // insert_subvector here to avoid DAG combining it back to a large
    // build_vector.
    SmallVector<SDValue> BuildVectorOps(Op->ops());
    unsigned NumOpElts = M1VT.getVectorMinNumElements();
    SDValue Vec = DAG.getUNDEF(ContainerVT);
    for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
      auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
      SDValue SubBV =
          DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
      SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
      unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
      Vec = DAG.getInsertSubvector(DL, Vec, SubBV, InsertIdx);
    }
    return convertFromScalableVector(VT, Vec, DAG, Subtarget);
  }
 
  // If we're about to resort to vslide1down (or stack usage), pack our
  // elements into the widest scalar type we can.  This will force a VL/VTYPE
  // toggle, but reduces the critical path, the number of vslide1down ops
  // required, and possibly enables scalar folds of the values.
  if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
    return Res;
 
  // For m1 vectors, if we have non-undef values in both halves of our vector,
  // split the vector into low and high halves, build them separately, then
  // use a vselect to combine them.  For long vectors, this cuts the critical
  // path of the vslide1down sequence in half, and gives us an opportunity
  // to special case each half independently.  Note that we don't change the
  // length of the sub-vectors here, so if both fallback to the generic
  // vslide1down path, we should be able to fold the vselect into the final
  // vslidedown (for the undef tail) for the first half w/ masking.
  unsigned NumElts = VT.getVectorNumElements();
  unsigned NumUndefElts =
      count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
  unsigned NumDefElts = NumElts - NumUndefElts;
  if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
      ContainerVT.bitsLE(RISCVTargetLowering::getM1VT(ContainerVT))) {
    SmallVector<SDValue> SubVecAOps, SubVecBOps;
    SmallVector<SDValue> MaskVals;
    SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
    SubVecAOps.reserve(NumElts);
    SubVecBOps.reserve(NumElts);
    for (const auto &[Idx, U] : enumerate(Op->ops())) {
      SDValue Elem = U.get();
      if (Idx < NumElts / 2) {
        SubVecAOps.push_back(Elem);
        SubVecBOps.push_back(UndefElem);
      } else {
        SubVecAOps.push_back(UndefElem);
        SubVecBOps.push_back(Elem);
      }
      bool SelectMaskVal = (Idx < NumElts / 2);
      MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
    }
    assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
           MaskVals.size() == NumElts);
 
    SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
    SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
    MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
    SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
    return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
  }
 
  // Cap the cost at a value linear to the number of elements in the vector.
  // The default lowering is to use the stack.  The vector store + scalar loads
  // is linear in VL.  However, at high lmuls vslide1down and vslidedown end up
  // being (at least) linear in LMUL.  As a result, using the vslidedown
  // lowering for every element ends up being VL*LMUL..
  // TODO: Should we be directly costing the stack alternative?  Doing so might
  // give us a more accurate upper bound.
  InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
 
  // TODO: unify with TTI getSlideCost.
  InstructionCost PerSlideCost = 1;
  switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
  default: break;
  case RISCVVType::LMUL_2:
    PerSlideCost = 2;
    break;
  case RISCVVType::LMUL_4:
    PerSlideCost = 4;
    break;
  case RISCVVType::LMUL_8:
    PerSlideCost = 8;
    break;
  }
 
  // TODO: Should we be using the build instseq then cost + evaluate scheme
  // we use for integer constants here?
  unsigned UndefCount = 0;
  for (const SDValue &V : Op->ops()) {
    if (V.isUndef()) {
      UndefCount++;
      continue;
    }
    if (UndefCount) {
      LinearBudget -= PerSlideCost;
      UndefCount = 0;
    }
    LinearBudget -= PerSlideCost;
  }
  if (UndefCount) {
    LinearBudget -= PerSlideCost;
  }
 
  if (LinearBudget < 0)
    return SDValue();
 
  assert((!VT.isFloatingPoint() ||
          VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
         "Illegal type which will result in reserved encoding");
 
  const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
 
  // General case: splat the first operand and slide other operands down one
  // by one to form a vector. Alternatively, if every operand is an
  // extraction from element 0 of a vector, we use that vector from the last
  // extraction as the start value and slide up instead of slide down. Such that
  // (1) we can avoid the initial splat (2) we can turn those vslide1up into
  // vslideup of 1 later and eliminate the vector to scalar movement, which is
  // something we cannot do with vslide1down/vslidedown.
  // Of course, using vslide1up/vslideup might increase the register pressure,
  // and that's why we conservatively limit to cases where every operand is an
  // extraction from the first element.
  SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
  SDValue EVec;
  bool SlideUp = false;
  auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
                       SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
    if (SlideUp)
      return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
                         Mask, VL, Policy);
    return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
                         Mask, VL, Policy);
  };
 
  // The reason we don't use all_of here is because we're also capturing EVec
  // from the last non-undef operand. If the std::execution_policy of the
  // underlying std::all_of is anything but std::sequenced_policy we might
  // capture the wrong EVec.
  for (SDValue V : Operands) {
    using namespace SDPatternMatch;
    SlideUp = V.isUndef() || sd_match(V, m_ExtractElt(m_Value(EVec), m_Zero()));
    if (!SlideUp)
      break;
  }
 
  // Do not slideup if the element type of EVec is different.
  if (SlideUp) {
    MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
    MVT ContainerEltVT = ContainerVT.getVectorElementType();
    if (EVecEltVT != ContainerEltVT)
      SlideUp = false;
  }
 
  if (SlideUp) {
    MVT EVecContainerVT = EVec.getSimpleValueType();
    // Make sure the original vector has scalable vector type.
    if (EVecContainerVT.isFixedLengthVector()) {
      EVecContainerVT =
          getContainerForFixedLengthVector(EVecContainerVT, Subtarget);
      EVec = convertToScalableVector(EVecContainerVT, EVec, DAG, Subtarget);
    }
 
    // Adapt EVec's type into ContainerVT.
    if (EVecContainerVT.getVectorMinNumElements() <
        ContainerVT.getVectorMinNumElements())
      EVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), EVec, 0);
    else
      EVec = DAG.getExtractSubvector(DL, ContainerVT, EVec, 0);
 
    // Reverse the elements as we're going to slide up from the last element.
    std::reverse(Operands.begin(), Operands.end());
  }
 
  SDValue Vec;
  UndefCount = 0;
  for (SDValue V : Operands) {
    if (V.isUndef()) {
      UndefCount++;
      continue;
    }
 
    // Start our sequence with either a TA splat or extract source in the
    // hopes that hardware is able to recognize there's no dependency on the
    // prior value of our temporary register.
    if (!Vec) {
      if (SlideUp) {
        Vec = EVec;
      } else {
        Vec = DAG.getSplatVector(VT, DL, V);
        Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
      }
 
      UndefCount = 0;
      continue;
    }
 
    if (UndefCount) {
      const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
      Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
                      VL);
      UndefCount = 0;
    }
 
    unsigned Opcode;
    if (VT.isFloatingPoint())
      Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
    else
      Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
 
    if (!VT.isFloatingPoint())
      V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
    Vec = DAG.getNode(Opcode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
                      V, Mask, VL);
  }
  if (UndefCount) {
    const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
    Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
                    VL);
  }
  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
 
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
                                   SDValue Lo, SDValue Hi, SDValue VL,
                                   SelectionDAG &DAG) {
  if (!Passthru)
    Passthru = DAG.getUNDEF(VT);
  if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
    int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
    int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
    // If Hi constant is all the same sign bit as Lo, lower this as a custom
    // node in order to try and match RVV vector/scalar instructions.
    if ((LoC >> 31) == HiC)
      return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
    // Use vmv.v.x with EEW=32.  Use either a vsetivli or vsetvli to change
    // VL.  This can temporarily increase VL if VL less than VLMAX.
    if (LoC == HiC) {
      SDValue NewVL;
      if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
        NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
      else
        NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
      MVT InterVT =
          MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
      auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
                                  DAG.getUNDEF(InterVT), Lo, NewVL);
      return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
    }
  }
 
  // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
  if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
      isa<ConstantSDNode>(Hi.getOperand(1)) &&
      Hi.getConstantOperandVal(1) == 31)
    return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
  // If the hi bits of the splat are undefined, then it's fine to just splat Lo
  // even if it might be sign extended.
  if (Hi.isUndef())
    return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
  // Fall back to a stack store and stride x0 vector load.
  return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
                     Hi, VL);
}
 
// Called by type legalization to handle splat of i64 on RV32.
// FIXME: We can optimize this when the type has sign or zero bits in one
// of the halves.
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
                                   SDValue Scalar, SDValue VL,
                                   SelectionDAG &DAG) {
  assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
  SDValue Lo, Hi;
  std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
  return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
}
 
// This function lowers a splat of a scalar operand Splat with the vector
// length VL. It ensures the final sequence is type legal, which is useful when
// lowering a splat after type legalization.
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
                                MVT VT, const SDLoc &DL, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  bool HasPassthru = Passthru && !Passthru.isUndef();
  if (!HasPassthru && !Passthru)
    Passthru = DAG.getUNDEF(VT);
 
  MVT EltVT = VT.getVectorElementType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  if (VT.isFloatingPoint()) {
    if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
        (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
      if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
          (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
        Scalar = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Scalar);
      else
        Scalar = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Scalar);
      MVT IVT = VT.changeVectorElementType(MVT::i16);
      Passthru = DAG.getNode(ISD::BITCAST, DL, IVT, Passthru);
      SDValue Splat =
          lowerScalarSplat(Passthru, Scalar, VL, IVT, DL, DAG, Subtarget);
      return DAG.getNode(ISD::BITCAST, DL, VT, Splat);
    }
    return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
  }
 
  // Simplest case is that the operand needs to be promoted to XLenVT.
  if (Scalar.getValueType().bitsLE(XLenVT)) {
    // If the operand is a constant, sign extend to increase our chances
    // of being able to use a .vi instruction. ANY_EXTEND would become a
    // a zero extend and the simm5 check in isel would fail.
    // FIXME: Should we ignore the upper bits in isel instead?
    unsigned ExtOpc =
        isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
    Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
    return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
  }
 
  assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
         "Unexpected scalar for splat lowering!");
 
  if (isOneConstant(VL) && isNullConstant(Scalar))
    return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
                       DAG.getConstant(0, DL, XLenVT), VL);
 
  // Otherwise use the more complicated splatting algorithm.
  return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
}
 
// This function lowers an insert of a scalar operand Scalar into lane
// 0 of the vector regardless of the value of VL.  The contents of the
// remaining lanes of the result vector are unspecified.  VL is assumed
// to be non-zero.
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
                                 const SDLoc &DL, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
 
  const MVT XLenVT = Subtarget.getXLenVT();
  SDValue Passthru = DAG.getUNDEF(VT);
 
  if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
      isNullConstant(Scalar.getOperand(1))) {
    SDValue ExtractedVal = Scalar.getOperand(0);
    // The element types must be the same.
    if (ExtractedVal.getValueType().getVectorElementType() ==
        VT.getVectorElementType()) {
      MVT ExtractedVT = ExtractedVal.getSimpleValueType();
      MVT ExtractedContainerVT = ExtractedVT;
      if (ExtractedContainerVT.isFixedLengthVector()) {
        ExtractedContainerVT =
            getContainerForFixedLengthVector(ExtractedContainerVT, Subtarget);
        ExtractedVal = convertToScalableVector(ExtractedContainerVT,
                                               ExtractedVal, DAG, Subtarget);
      }
      if (ExtractedContainerVT.bitsLE(VT))
        return DAG.getInsertSubvector(DL, Passthru, ExtractedVal, 0);
      return DAG.getExtractSubvector(DL, VT, ExtractedVal, 0);
    }
  }
 
  if (VT.isFloatingPoint())
    return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
                       VL);
 
  // Avoid the tricky legalization cases by falling back to using the
  // splat code which already handles it gracefully.
  if (!Scalar.getValueType().bitsLE(XLenVT))
    return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
                            DAG.getConstant(1, DL, XLenVT),
                            VT, DL, DAG, Subtarget);
 
  // If the operand is a constant, sign extend to increase our chances
  // of being able to use a .vi instruction. ANY_EXTEND would become a
  // a zero extend and the simm5 check in isel would fail.
  // FIXME: Should we ignore the upper bits in isel instead?
  unsigned ExtOpc =
      isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
  Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
  return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
                     VL);
}
 
/// If concat_vector(V1,V2) could be folded away to some existing
/// vector source, return it.  Note that the source may be larger
/// than the requested concat_vector (i.e. a extract_subvector
/// might be required.)
static SDValue foldConcatVector(SDValue V1, SDValue V2) {
  EVT VT = V1.getValueType();
  assert(VT == V2.getValueType() && "argument types must match");
  // Both input must be extracts.
  if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
      V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
    return SDValue();
 
  // Extracting from the same source.
  SDValue Src = V1.getOperand(0);
  if (Src != V2.getOperand(0) ||
      VT.isScalableVector() != Src.getValueType().isScalableVector())
    return SDValue();
 
  // The extracts must extract the two halves of the source.
  if (V1.getConstantOperandVal(1) != 0 ||
      V2.getConstantOperandVal(1) != VT.getVectorMinNumElements())
    return SDValue();
 
  return Src;
}
 
// Can this shuffle be performed on exactly one (possibly larger) input?
static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
 
  if (V2.isUndef())
    return V1;
 
  unsigned NumElts = VT.getVectorNumElements();
  // Src needs to have twice the number of elements.
  // TODO: Update shuffle lowering to add the extract subvector
  if (SDValue Src = foldConcatVector(V1, V2);
      Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
    return Src;
 
  return SDValue();
}
 
static unsigned getLMULOctuple(MVT ContainerVT) {
  assert(ContainerVT.isScalableVector() && "Expected scalable vector type");
  unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
  assert(isPowerOf2_32(MinSize) && MinSize >= 8 && MinSize <= 512 &&
         "Unexpected LMUL");
  return MinSize / (RISCV::RVVBitsPerBlock / 8);
}
 
static bool
isLegalVTForZvzipDeinterleavedOperand(MVT VT, const RISCVSubtarget &Subtarget) {
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  return RISCVTargetLowering::getLMUL(ContainerVT) != RISCVVType::LMUL_8;
}
 
static bool
isLegalVTForZvzipInterleavedOperand(MVT VT, const RISCVSubtarget &Subtarget) {
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    // VT is the interleaved result type, but lowerZvzipVZIP maps each
    // half-sized input to a scalable container before doubling that container
    // for the result. Mirror that order here because the minimum scalable
    // container size means mapping VT directly is not necessarily equivalent.
    // For example, interleaving two <2 x i32> inputs at VLEN=128 requires an
    // m2 result container, while mapping the <4 x i32> result directly yields
    // m1.
    ContainerVT = getContainerForFixedLengthVector(
        VT.getHalfNumVectorElementsVT(), Subtarget);
    ContainerVT = ContainerVT.getDoubleNumVectorElementsVT();
  }
  unsigned EltBits = VT.getScalarSizeInBits();
  unsigned LMULOctuple = getLMULOctuple(ContainerVT);
  // Perform 2 * SEW <= LMUL * min(ELEN, VLEN) check.
  return EltBits * 16 <= LMULOctuple * std::min(Subtarget.getELen(),
                                                Subtarget.getRealMinVLen());
}
 
/// Is this shuffle interleaving contiguous elements from one vector into the
/// even elements and contiguous elements from another vector into the odd
/// elements. \p EvenSrc will contain the element that should be in the first
/// even element. \p OddSrc will contain the element that should be in the first
/// odd element. These can be the first element in a source or the element half
/// way through the source.
static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
                                int &OddSrc, const RISCVSubtarget &Subtarget) {
  // We need to be able to widen elements to the next larger integer type or
  // use the vzip instruction at e64.
  if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
    if (!Subtarget.hasStdExtZvzip())
      return false;
    if (!isLegalVTForZvzipInterleavedOperand(VT, Subtarget))
      return false;
  }
 
  int Size = Mask.size();
  int NumElts = VT.getVectorNumElements();
  assert(Size == (int)NumElts && "Unexpected mask size");
 
  SmallVector<unsigned, 2> StartIndexes;
  if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
    return false;
 
  EvenSrc = StartIndexes[0];
  OddSrc = StartIndexes[1];
 
  // One source should be low half of first vector.
  if (EvenSrc != 0 && OddSrc != 0)
    return false;
 
  // Subvectors will be subtracted from either at the start of the two input
  // vectors, or at the start and middle of the first vector if it's an unary
  // interleave.
  // In both cases, HalfNumElts will be extracted.
  // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
  // we'll create an illegal extract_subvector.
  // FIXME: We could support other values using a slidedown first.
  int HalfNumElts = NumElts / 2;
  return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
}
 
/// Is this mask representing a masked combination of two slides?
static bool isMaskedSlidePair(ArrayRef<int> Mask,
                              std::array<std::pair<int, int>, 2> &SrcInfo) {
  if (!llvm::isMaskedSlidePair(Mask, Mask.size(), SrcInfo))
    return false;
 
  // Avoid matching vselect idioms
  if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
    return false;
  // Prefer vslideup as the second instruction, and identity
  // only as the initial instruction.
  if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
      SrcInfo[1].second == 0)
    std::swap(SrcInfo[0], SrcInfo[1]);
  assert(SrcInfo[0].first != -1 && "Must find one slide");
  return true;
}
 
// Exactly matches the semantics of a previously existing custom matcher
// to allow migration to new matcher without changing output.
static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
                            unsigned NumElts) {
  if (SrcInfo[1].first == -1)
    return true;
  return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
         SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
}
 
// Lower a deinterleave shuffle to SRL and TRUNC.  Factor must be
// 2, 4, 8 and the integer type Factor-times larger than VT's
// element type must be a legal element type.
// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
//                          -> [p, q, r, s] (Factor=2, Index=1)
static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
                                            SDValue Src, unsigned Factor,
                                            unsigned Index, SelectionDAG &DAG) {
  unsigned EltBits = VT.getScalarSizeInBits();
  ElementCount SrcEC = Src.getValueType().getVectorElementCount();
  MVT WideSrcVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor),
                                   SrcEC.divideCoefficientBy(Factor));
  MVT ResVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits),
                               SrcEC.divideCoefficientBy(Factor));
  Src = DAG.getBitcast(WideSrcVT, Src);
 
  unsigned Shift = Index * EltBits;
  SDValue Res = DAG.getNode(ISD::SRL, DL, WideSrcVT, Src,
                            DAG.getConstant(Shift, DL, WideSrcVT));
  Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT, Res);
  MVT CastVT = ResVT.changeVectorElementType(VT.getVectorElementType());
  Res = DAG.getBitcast(CastVT, Res);
  return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
}
 
/// Match a single source shuffle which is an identity except that some
/// particular element is repeated.  This can be lowered as a masked
/// vrgather.vi/vx.  Note that the two source form of this is handled
/// by the recursive splitting logic and doesn't need special handling.
static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
                                               const RISCVSubtarget &Subtarget,
                                               SelectionDAG &DAG) {
 
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  SDValue V1 = SVN->getOperand(0);
  assert(SVN->getOperand(1).isUndef());
  ArrayRef<int> Mask = SVN->getMask();
  const unsigned NumElts = VT.getVectorNumElements();
  MVT XLenVT = Subtarget.getXLenVT();
 
  std::optional<int> SplatIdx;
  for (auto [I, M] : enumerate(Mask)) {
    if (M == -1 || I == (unsigned)M)
      continue;
    if (SplatIdx && *SplatIdx != M)
      return SDValue();
    SplatIdx = M;
  }
 
  if (!SplatIdx)
    return SDValue();
 
  SmallVector<SDValue> MaskVals;
  for (int MaskIndex : Mask) {
    bool SelectMaskVal = MaskIndex == *SplatIdx;
    MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
  }
  assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
  SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
  SDValue Splat = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT),
                                       SmallVector<int>(NumElts, *SplatIdx));
  return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, Splat, V1);
}
 
// Lower the following shuffle to vslidedown.
// a)
// t49: v8i8 = extract_subvector t13, Constant:i64<0>
// t109: v8i8 = extract_subvector t13, Constant:i64<8>
// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
// b)
// t69: v16i16 = extract_subvector t68, Constant:i64<0>
// t23: v8i16 = extract_subvector t69, Constant:i64<0>
// t29: v4i16 = extract_subvector t23, Constant:i64<4>
// t26: v8i16 = extract_subvector t69, Constant:i64<8>
// t30: v4i16 = extract_subvector t26, Constant:i64<0>
// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
                                               SDValue V1, SDValue V2,
                                               ArrayRef<int> Mask,
                                               const RISCVSubtarget &Subtarget,
                                               SelectionDAG &DAG) {
  auto findNonEXTRACT_SUBVECTORParent =
      [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
    uint64_t Offset = 0;
    while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
           // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
           // a scalable vector. But we don't want to match the case.
           Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
      Offset += Parent.getConstantOperandVal(1);
      Parent = Parent.getOperand(0);
    }
    return std::make_pair(Parent, Offset);
  };
 
  auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
  auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
 
  // Extracting from the same source.
  SDValue Src = V1Src;
  if (Src != V2Src)
    return SDValue();
 
  // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
  SmallVector<int, 16> NewMask(Mask);
  for (size_t i = 0; i != NewMask.size(); ++i) {
    if (NewMask[i] == -1)
      continue;
 
    if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
      NewMask[i] = NewMask[i] + V1IndexOffset;
    } else {
      // Minus NewMask.size() is needed. Otherwise, the b case would be
      // <5,6,7,12> instead of <5,6,7,8>.
      NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
    }
  }
 
  // First index must be known and non-zero. It will be used as the slidedown
  // amount.
  if (NewMask[0] <= 0)
    return SDValue();
 
  // NewMask is also continuous.
  for (unsigned i = 1; i != NewMask.size(); ++i)
    if (NewMask[i - 1] + 1 != NewMask[i])
      return SDValue();
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT SrcVT = Src.getSimpleValueType();
  MVT ContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
  auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
  SDValue Slidedown =
      getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
                    convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
                    DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
  return DAG.getExtractSubvector(
      DL, VT, convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), 0);
}
 
// Because vslideup leaves the destination elements at the start intact, we can
// use it to perform shuffles that insert subvectors:
//
// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
// ->
// vsetvli zero, 8, e8, mf2, ta, ma
// vslideup.vi v8, v9, 4
//
// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
// ->
// vsetvli zero, 5, e8, mf2, tu, ma
// vslideup.v1 v8, v9, 2
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
                                             SDValue V1, SDValue V2,
                                             ArrayRef<int> Mask,
                                             const RISCVSubtarget &Subtarget,
                                             SelectionDAG &DAG) {
  unsigned NumElts = VT.getVectorNumElements();
  int NumSubElts, Index;
  if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumElts, NumSubElts,
                                                Index))
    return SDValue();
 
  bool OpsSwapped = Mask[Index] < (int)NumElts;
  SDValue InPlace = OpsSwapped ? V2 : V1;
  SDValue ToInsert = OpsSwapped ? V1 : V2;
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
  // We slide up by the index that the subvector is being inserted at, and set
  // VL to the index + the number of elements being inserted.
  unsigned Policy =
      RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
  // If the we're adding a suffix to the in place vector, i.e. inserting right
  // up to the very end of it, then we don't actually care about the tail.
  if (NumSubElts + Index >= (int)NumElts)
    Policy |= RISCVVType::TAIL_AGNOSTIC;
 
  InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
  ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
  SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
 
  SDValue Res;
  // If we're inserting into the lowest elements, use a tail undisturbed
  // vmv.v.v.
  if (Index == 0)
    Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
                      VL);
  else
    Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
                      DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
  return convertFromScalableVector(VT, Res, DAG, Subtarget);
}
 
// A shuffle of shuffles where the final data only is drawn from 2 input ops
// can be compressed into a single shuffle
static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
                                         const RISCVSubtarget &Subtarget,
                                         SelectionDAG &DAG) {
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
 
  if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
      V2.getOpcode() != ISD::VECTOR_SHUFFLE)
    return SDValue();
 
  if (!V1.hasOneUse() || !V2.hasOneUse())
    return SDValue();
 
  ArrayRef<int> Mask = SVN->getMask();
  ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(V1.getNode())->getMask();
  ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(V2.getNode())->getMask();
  unsigned NumElts = Mask.size();
  SmallVector<int> NewMask(NumElts, -1);
  for (unsigned Idx : seq<unsigned>(NumElts)) {
    int Lane = Mask[Idx];
    // Don't assign if poison
    if (Lane == -1)
      continue;
    int OrigLane;
    bool SecondOp = false;
    if ((unsigned)Lane < NumElts) {
      OrigLane = V1Mask[Lane];
    } else {
      OrigLane = V2Mask[Lane - NumElts];
      SecondOp = true;
    }
    if (OrigLane == -1)
      continue;
    // Don't handle if shuffling from a second operand
    if ((unsigned)OrigLane >= NumElts)
      return SDValue();
    if (SecondOp)
      OrigLane += NumElts;
    NewMask[Idx] = OrigLane;
  }
 
  EVT VT = SVN->getValueType(0);
  SDLoc DL(SVN);
 
  return DAG.getVectorShuffle(VT, DL, V1->getOperand(0), V2->getOperand(0),
                              NewMask);
}
 
/// Match v(f)slide1up/down idioms.  These operations involve sliding
/// N-1 elements to make room for an inserted scalar at one end.
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
                                            SDValue V1, SDValue V2,
                                            ArrayRef<int> Mask,
                                            const RISCVSubtarget &Subtarget,
                                            SelectionDAG &DAG) {
  bool OpsSwapped = false;
  if (!isa<BuildVectorSDNode>(V1)) {
    if (!isa<BuildVectorSDNode>(V2))
      return SDValue();
    std::swap(V1, V2);
    OpsSwapped = true;
  }
  SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
  if (!Splat)
    return SDValue();
 
  // Return true if the mask could describe a slide of Mask.size() - 1
  // elements from concat_vector(V1, V2)[Base:] to [Offset:].
  auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
    const unsigned S = (Offset > 0) ? 0 : -Offset;
    const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
    for (unsigned i = S; i != E; ++i)
      if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
        return false;
    return true;
  };
 
  const unsigned NumElts = VT.getVectorNumElements();
  bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
  if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
    return SDValue();
 
  const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
  // Inserted lane must come from splat, undef scalar is legal but not profitable.
  if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
    return SDValue();
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
  // vslide1{down,up}.vx instead.
  if ((VT.getVectorElementType() == MVT::bf16 &&
       !Subtarget.hasVInstructionsBF16()) ||
      (VT.getVectorElementType() == MVT::f16 &&
       !Subtarget.hasVInstructionsF16())) {
    MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
    Splat =
        DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Splat);
    V2 = DAG.getBitcast(
        IntVT, convertToScalableVector(ContainerVT, V2, DAG, Subtarget));
    SDValue Vec = DAG.getNode(
        IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
        IntVT, DAG.getUNDEF(IntVT), V2, Splat, TrueMask, VL);
    Vec = DAG.getBitcast(ContainerVT, Vec);
    return convertFromScalableVector(VT, Vec, DAG, Subtarget);
  }
 
  auto OpCode = IsVSlidedown ?
    (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
    (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
  if (!VT.isFloatingPoint())
    Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
  auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
                         DAG.getUNDEF(ContainerVT),
                         convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
                         Splat, TrueMask, VL);
  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
 
/// Match a mask which "spreads" the leading elements of a vector evenly
/// across the result.  Factor is the spread amount, and Index is the
/// offset applied.  (on success, Index < Factor)  This is the inverse
/// of a deinterleave with the same Factor and Index.  This is analogous
/// to an interleave, except that all but one lane is undef.
bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
                                       unsigned &Index) {
  SmallVector<bool> LaneIsUndef(Factor, true);
  for (unsigned i = 0; i < Mask.size(); i++)
    LaneIsUndef[i % Factor] &= (Mask[i] == -1);
 
  bool Found = false;
  for (unsigned i = 0; i < Factor; i++) {
    if (LaneIsUndef[i])
      continue;
    if (Found)
      return false;
    Index = i;
    Found = true;
  }
  if (!Found)
    return false;
 
  for (unsigned i = 0; i < Mask.size() / Factor; i++) {
    unsigned j = i * Factor + Index;
    if (Mask[j] != -1 && (unsigned)Mask[j] != i)
      return false;
  }
  return true;
}
 
static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
                               const SDLoc &DL, SelectionDAG &DAG,
                               const RISCVSubtarget &Subtarget) {
  assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
  assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
 
  MVT VT = Op0.getSimpleValueType();
  MVT IntVT = VT.changeVectorElementTypeToInteger();
  Op0 = DAG.getBitcast(IntVT, Op0);
  Op1 = DAG.getBitcast(IntVT, Op1);
 
  MVT ContainerVT = IntVT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
  }
 
  MVT InnerVT = ContainerVT;
  auto [Mask, VL] = getDefaultVLOps(IntVT, InnerVT, DL, DAG, Subtarget);
 
  SDValue Passthru = DAG.getUNDEF(InnerVT);
  SDValue Res = DAG.getNode(Opc, DL, InnerVT, Op0, Op1, Passthru, Mask, VL);
  if (IntVT.isFixedLengthVector())
    Res = convertFromScalableVector(IntVT, Res, DAG, Subtarget);
  Res = DAG.getBitcast(VT, Res);
  return Res;
}
 
static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
                              SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
  MVT VT = Op0.getSimpleValueType();
  MVT IntVT = VT.changeVectorElementTypeToInteger();
  Op0 = DAG.getBitcast(IntVT, Op0);
  Op1 = DAG.getBitcast(IntVT, Op1);
  MVT ContainerVT = IntVT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
  }
  MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
  MVT ResIntVT = IntVT.getDoubleNumVectorElementsVT();
  auto [Mask, VL] = getDefaultVLOps(ResIntVT, ResVT, DL, DAG, Subtarget);
  SDValue Passthru = DAG.getUNDEF(ResVT);
  SDValue Res =
      DAG.getNode(RISCVISD::VZIP_VL, DL, ResVT, Op0, Op1, Passthru, Mask, VL);
  if (IntVT.isFixedLengthVector())
    Res = convertFromScalableVector(IntVT.getDoubleNumVectorElementsVT(), Res,
                                    DAG, Subtarget);
  Res = DAG.getBitcast(VT.getDoubleNumVectorElementsVT(), Res);
  return Res;
}
 
static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
                                SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
  MVT VT = Op.getSimpleValueType();
  assert(VT.getVectorMinNumElements() >= 2);
 
  MVT IntVT = VT.changeVectorElementTypeToInteger();
  Op = DAG.getBitcast(IntVT, Op);
  MVT ContainerVT = IntVT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
    // Ensure that halving the source container produces a legal result type.
    // In particular, E64 has no fractional LMUL types, and some Zve profiles
    // do not legalize the smallest fractional LMUL type.
    if (ContainerVT.getVectorMinNumElements() == 1 ||
        !Subtarget.getTargetLowering()->isTypeLegal(
            ContainerVT.getHalfNumVectorElementsVT()))
      ContainerVT = ContainerVT.getDoubleNumVectorElementsVT();
    Op = convertToScalableVector(ContainerVT, Op, DAG, Subtarget);
  }
 
  MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
  MVT HalfVT = VT.getHalfNumVectorElementsVT();
  MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
  SDValue VL = getDefaultVLOps(HalfIntVT, ResVT, DL, DAG, Subtarget).second;
  SDValue Passthru = DAG.getUNDEF(ResVT);
  SDValue Res = DAG.getNode(Opc, DL, ResVT, Op, Passthru, VL);
  if (HalfIntVT.isFixedLengthVector())
    Res = convertFromScalableVector(HalfIntVT, Res, DAG, Subtarget);
  Res = DAG.getBitcast(HalfVT, Res);
  return Res;
}
 
// Given a vector a, b, c, d return a vector Factor times longer
// with Factor-1 undef's between elements. Ex:
//   a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
//   undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
                                 const SDLoc &DL, SelectionDAG &DAG) {
 
  MVT VT = V.getSimpleValueType();
  unsigned EltBits = VT.getScalarSizeInBits();
  ElementCount EC = VT.getVectorElementCount();
  V = DAG.getBitcast(VT.changeTypeToInteger(), V);
 
  MVT WideVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor), EC);
 
  SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, V);
  // TODO: On rv32, the constant becomes a splat_vector_parts which does not
  // allow the SHL to fold away if Index is 0.
  if (Index != 0)
    Result = DAG.getNode(ISD::SHL, DL, WideVT, Result,
                         DAG.getConstant(EltBits * Index, DL, WideVT));
  // Make sure to use original element type
  MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(), EC * Factor);
  return DAG.getBitcast(ResultVT, Result);
}
 
// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
// to create an interleaved vector of <[vscale x] n*2 x ty>.
// This requires that the size of ty is less than the subtarget's maximum ELEN.
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
                                     const SDLoc &DL, SelectionDAG &DAG,
                                     const RISCVSubtarget &Subtarget) {
 
  // FIXME: Not only does this optimize the code, it fixes some correctness
  // issues because MIR does not have freeze.
  if (EvenV.isUndef())
    return getWideningSpread(OddV, 2, 1, DL, DAG);
  if (OddV.isUndef())
    return getWideningSpread(EvenV, 2, 0, DL, DAG);
 
  MVT VecVT = EvenV.getSimpleValueType();
  MVT VecContainerVT = VecVT; // <vscale x n x ty>
  // Convert fixed vectors to scalable if needed
  if (VecContainerVT.isFixedLengthVector()) {
    VecContainerVT = getContainerForFixedLengthVector(VecVT, Subtarget);
    EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
    OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
  }
 
  assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
 
  // We're working with a vector of the same size as the resulting
  // interleaved vector, but with half the number of elements and
  // twice the SEW (Hence the restriction on not using the maximum
  // ELEN)
  MVT WideVT =
      MVT::getVectorVT(MVT::getIntegerVT(VecVT.getScalarSizeInBits() * 2),
                       VecVT.getVectorElementCount());
  MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
  if (WideContainerVT.isFixedLengthVector())
    WideContainerVT = getContainerForFixedLengthVector(WideVT, Subtarget);
 
  // Bitcast the input vectors to integers in case they are FP
  VecContainerVT = VecContainerVT.changeTypeToInteger();
  EvenV = DAG.getBitcast(VecContainerVT, EvenV);
  OddV = DAG.getBitcast(VecContainerVT, OddV);
 
  auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
  SDValue Passthru = DAG.getUNDEF(WideContainerVT);
 
  SDValue Interleaved;
  if (Subtarget.hasStdExtZvbb()) {
    // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
    SDValue OffsetVec =
        DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
    Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
                              OffsetVec, Passthru, Mask, VL);
    Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
                              Interleaved, EvenV, Passthru, Mask, VL);
  } else {
    // FIXME: We should freeze the odd vector here. We already handled the case
    // of provably undef/poison above.
 
    // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
    // vwaddu.vv
    Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
                              OddV, Passthru, Mask, VL);
 
    // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
    SDValue AllOnesVec = DAG.getSplatVector(
        VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
    SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
                                  OddV, AllOnesVec, Passthru, Mask, VL);
 
    // Add the two together so we get
    //   (OddV * 0xff...ff) + (OddV + EvenV)
    // = (OddV * 0x100...00) + EvenV
    // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
    // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
    Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
                              Interleaved, OddsMul, Passthru, Mask, VL);
  }
 
  // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
  MVT ResultContainerVT = MVT::getVectorVT(
      VecVT.getVectorElementType(), // Make sure to use original type
      VecContainerVT.getVectorElementCount() * 2);
  Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
 
  // Convert back to a fixed vector if needed
  MVT ResultVT = MVT::getVectorVT(VecVT.getVectorElementType(),
                                  VecVT.getVectorElementCount() * 2);
  if (ResultVT.isFixedLengthVector())
    Interleaved =
        convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
 
  return Interleaved;
}
 
// If we have a vector of bits that we want to reverse, we can use a vbrev on a
// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
                                      SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  SDValue V = SVN->getOperand(0);
  unsigned NumElts = VT.getVectorNumElements();
 
  assert(VT.getVectorElementType() == MVT::i1);
 
  if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
                                        SVN->getMask().size()) ||
      !SVN->getOperand(1).isUndef())
    return SDValue();
 
  unsigned ViaEltSize = std::max((uint64_t)8, PowerOf2Ceil(NumElts));
  EVT ViaVT = EVT::getVectorVT(
      *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), ViaEltSize), 1);
  EVT ViaBitVT =
      EVT::getVectorVT(*DAG.getContext(), MVT::i1, ViaVT.getScalarSizeInBits());
 
  // If we don't have zvbb or the larger element type > ELEN, the operation will
  // be illegal.
  if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
                                                               ViaVT) ||
      !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT))
    return SDValue();
 
  // If the bit vector doesn't fit exactly into the larger element type, we need
  // to insert it into the larger vector and then shift up the reversed bits
  // afterwards to get rid of the gap introduced.
  if (ViaEltSize > NumElts)
    V = DAG.getInsertSubvector(DL, DAG.getUNDEF(ViaBitVT), V, 0);
 
  SDValue Res =
      DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
 
  // Shift up the reversed bits if the vector didn't exactly fit into the larger
  // element type.
  if (ViaEltSize > NumElts)
    Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
                      DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
 
  Res = DAG.getBitcast(ViaBitVT, Res);
 
  if (ViaEltSize > NumElts)
    Res = DAG.getExtractSubvector(DL, VT, Res, 0);
  return Res;
}
 
static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
                             const RISCVSubtarget &Subtarget,
                             MVT &RotateVT, unsigned &RotateAmt) {
  unsigned NumElts = VT.getVectorNumElements();
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  unsigned NumSubElts;
  if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, 2,
                                          NumElts, NumSubElts, RotateAmt))
    return false;
  RotateVT = MVT::getVectorVT(MVT::getIntegerVT(EltSizeInBits * NumSubElts),
                              NumElts / NumSubElts);
 
  // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
  return Subtarget.getTargetLowering()->isTypeLegal(RotateVT);
}
 
// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
                                           SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  SDLoc DL(SVN);
 
  EVT VT = SVN->getValueType(0);
  unsigned RotateAmt;
  MVT RotateVT;
  if (!isLegalBitRotate(SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
    return SDValue();
 
  SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
 
  SDValue Rotate;
  // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
  // so canonicalize to vrev8.
  if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
    Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
  else
    Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
                         DAG.getConstant(RotateAmt, DL, RotateVT));
 
  return DAG.getBitcast(VT, Rotate);
}
 
// If compiling with an exactly known VLEN, see if we can split a
// shuffle on m2 or larger into a small number of m1 sized shuffles
// which write each destination registers exactly once.
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
                                            SelectionDAG &DAG,
                                            const RISCVSubtarget &Subtarget) {
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  ArrayRef<int> Mask = SVN->getMask();
 
  // If we don't know exact data layout, not much we can do.  If this
  // is already m1 or smaller, no point in splitting further.
  const auto VLen = Subtarget.getRealVLen();
  if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
    return SDValue();
 
  // Avoid picking up bitrotate patterns which we have a linear-in-lmul
  // expansion for.
  unsigned RotateAmt;
  MVT RotateVT;
  if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
    return SDValue();
 
  MVT ElemVT = VT.getVectorElementType();
  unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
 
  EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
  MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
  assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
  unsigned NumOpElts = M1VT.getVectorMinNumElements();
  unsigned NumElts = ContainerVT.getVectorMinNumElements();
  unsigned NumOfSrcRegs = NumElts / NumOpElts;
  unsigned NumOfDestRegs = NumElts / NumOpElts;
  // The following semantically builds up a fixed length concat_vector
  // of the component shuffle_vectors.  We eagerly lower to scalable here
  // to avoid DAG combining it back to a large shuffle_vector again.
  V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
  V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
  SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
      Operands;
  processShuffleMasks(
      Mask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs,
      [&]() { Operands.emplace_back(); },
      [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
        Operands.emplace_back().emplace_back(SrcVecIdx, UINT_MAX,
                                             SmallVector<int>(SrcSubMask));
      },
      [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
        if (NewReg)
          Operands.emplace_back();
        Operands.back().emplace_back(Idx1, Idx2, SmallVector<int>(SrcSubMask));
      });
  assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
  // Note: check that we do not emit too many shuffles here to prevent code
  // size explosion.
  // TODO: investigate, if it can be improved by extra analysis of the masks to
  // check if the code is more profitable.
  unsigned NumShuffles = std::accumulate(
      Operands.begin(), Operands.end(), 0u,
      [&](unsigned N,
          ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
        if (Data.empty())
          return N;
        N += Data.size();
        for (const auto &P : Data) {
          unsigned Idx2 = std::get<1>(P);
          ArrayRef<int> Mask = std::get<2>(P);
          if (Idx2 != UINT_MAX)
            ++N;
          else if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
            --N;
        }
        return N;
      });
  if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
      (NumOfDestRegs <= 2 && NumShuffles >= 4))
    return SDValue();
  auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
    SDValue SubVec = DAG.getExtractSubvector(DL, M1VT, SrcVec, ExtractIdx);
    SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget);
    return SubVec;
  };
  auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
                                        ArrayRef<int> Mask) {
    SDValue SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec1, SubVec2, Mask);
    return SubVec;
  };
  SDValue Vec = DAG.getUNDEF(ContainerVT);
  for (auto [I, Data] : enumerate(Operands)) {
    if (Data.empty())
      continue;
    SmallDenseMap<unsigned, SDValue, 4> Values;
    for (unsigned I : seq<unsigned>(Data.size())) {
      const auto &[Idx1, Idx2, _] = Data[I];
      // If the shuffle contains permutation of odd number of elements,
      // Idx1 might be used already in the first iteration.
      //
      // Idx1 = shuffle Idx1, Idx2
      // Idx1 = shuffle Idx1, Idx3
      SDValue &V = Values.try_emplace(Idx1).first->getSecond();
      if (!V)
        V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
                         (Idx1 % NumOfSrcRegs) * NumOpElts);
      if (Idx2 != UINT_MAX) {
        SDValue &V = Values.try_emplace(Idx2).first->getSecond();
        if (!V)
          V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
                           (Idx2 % NumOfSrcRegs) * NumOpElts);
      }
    }
    SDValue V;
    for (const auto &[Idx1, Idx2, Mask] : Data) {
      SDValue V1 = Values.at(Idx1);
      SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Idx2);
      V = PerformShuffle(V1, V2, Mask);
      Values[Idx1] = V;
    }
 
    unsigned InsertIdx = I * NumOpElts;
    V = convertToScalableVector(M1VT, V, DAG, Subtarget);
    Vec = DAG.getInsertSubvector(DL, Vec, V, InsertIdx);
  }
  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
 
// Matches a subset of compress masks with a contiguous prefix of output
// elements.  This could be extended to allow gaps by deciding which
// source elements to spuriously demand.
static bool isCompressMask(ArrayRef<int> Mask) {
  int Last = -1;
  bool SawUndef = false;
  for (const auto &[Idx, M] : enumerate(Mask)) {
    if (M == -1) {
      SawUndef = true;
      continue;
    }
    if (SawUndef)
      return false;
    if (Idx > (unsigned)M)
      return false;
    if (M <= Last)
      return false;
    Last = M;
  }
  return true;
}
 
/// Given a shuffle where the indices are disjoint between the two sources,
/// e.g.:
///
/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
///
/// Merge the two sources into one and do a single source shuffle:
///
/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
///
/// A vselect will either be merged into a masked instruction or be lowered as a
/// vmerge.vvm, which is cheaper than a vrgather.vv.
static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
                                           SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  MVT VT = SVN->getSimpleValueType(0);
  MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(SVN);
 
  const ArrayRef<int> Mask = SVN->getMask();
 
  // Work out which source each lane will come from.
  SmallVector<int, 16> Srcs(Mask.size(), -1);
 
  for (int Idx : Mask) {
    if (Idx == -1)
      continue;
    unsigned SrcIdx = Idx % Mask.size();
    int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
    if (Srcs[SrcIdx] == -1)
      // Mark this source as using this lane.
      Srcs[SrcIdx] = Src;
    else if (Srcs[SrcIdx] != Src)
      // The other source is using this lane: not disjoint.
      return SDValue();
  }
 
  SmallVector<SDValue> SelectMaskVals;
  for (int Lane : Srcs) {
    if (Lane == -1)
      SelectMaskVals.push_back(DAG.getUNDEF(XLenVT));
    else
      SelectMaskVals.push_back(DAG.getConstant(Lane ? 0 : 1, DL, XLenVT));
  }
  MVT MaskVT = VT.changeVectorElementType(MVT::i1);
  SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, SelectMaskVals);
  SDValue Select = DAG.getNode(ISD::VSELECT, DL, VT, SelectMask,
                               SVN->getOperand(0), SVN->getOperand(1));
 
  // Move all indices relative to the first source.
  SmallVector<int> NewMask(Mask.size());
  for (unsigned I = 0; I < Mask.size(); I++) {
    if (Mask[I] == -1)
      NewMask[I] = -1;
    else
      NewMask[I] = Mask[I] % Mask.size();
  }
 
  return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
}
 
/// Is this mask local (i.e. elements only move within their local span), and
/// repeating (that is, the same rearrangement is being done within each span)?
static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
  // Require a prefix from the original mask until the consumer code
  // is adjusted to rewrite the mask instead of just taking a prefix.
  for (auto [I, M] : enumerate(Mask)) {
    if (M == -1)
      continue;
    if ((M / Span) != (int)(I / Span))
      return false;
    int SpanIdx = I % Span;
    int Expected = M % Span;
    if (Mask[SpanIdx] != Expected)
      return false;
  }
  return true;
}
 
/// Is this mask only using elements from the first span of the input?
static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
  return all_of(Mask, [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
}
 
/// Return true for a mask which performs an arbitrary shuffle within the first
/// span, and then repeats that same result across all remaining spans.  Note
/// that this doesn't check if all the inputs come from a single span!
static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
  // Require a prefix from the original mask until the consumer code
  // is adjusted to rewrite the mask instead of just taking a prefix.
  for (auto [I, M] : enumerate(Mask)) {
    if (M == -1)
      continue;
    int SpanIdx = I % Span;
    if (Mask[SpanIdx] != M)
      return false;
  }
  return true;
}
 
/// Try to widen element type to get a new mask value for a better permutation
/// sequence.  This doesn't try to inspect the widened mask for profitability;
/// we speculate the widened form is equal or better.  This has the effect of
/// reducing mask constant sizes - allowing cheaper materialization sequences
/// - and index sequence sizes - reducing register pressure and materialization
/// cost, at the cost of (possibly) an extra VTYPE toggle.
static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT ScalarVT = VT.getVectorElementType();
  unsigned ElementSize = ScalarVT.getFixedSizeInBits();
  SDValue V0 = Op.getOperand(0);
  SDValue V1 = Op.getOperand(1);
  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
 
  // Avoid wasted work leading to isTypeLegal check failing below
  if (ElementSize > 32)
    return SDValue();
 
  SmallVector<int, 8> NewMask;
  if (!widenShuffleMaskElts(Mask, NewMask))
    return SDValue();
 
  MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(ElementSize * 2)
                                      : MVT::getIntegerVT(ElementSize * 2);
  MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
  if (!DAG.getTargetLoweringInfo().isTypeLegal(NewVT))
    return SDValue();
  V0 = DAG.getBitcast(NewVT, V0);
  V1 = DAG.getBitcast(NewVT, V1);
  return DAG.getBitcast(VT, DAG.getVectorShuffle(NewVT, DL, V0, V1, NewMask));
}
 
// Match an interleave shuffle that forms a P-extension packed zip:
//   <a0, b0, a1, b1, ...> -> zip*p/wzip*p
static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
                                         const RISCVSubtarget &Subtarget,
                                         SelectionDAG &DAG) {
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  unsigned NumElts = VT.getVectorNumElements();
  ArrayRef<int> Mask = SVN->getMask();
 
  if (VT != MVT::v8i8 && VT != MVT::v4i16)
    return SDValue();
 
  SmallVector<unsigned, 2> StartIndexes;
  if (!V2.isUndef() &&
      ShuffleVectorInst::isInterleaveMask(Mask, 2, NumElts * 2, StartIndexes)) {
    unsigned EvenSrc = StartIndexes[0];
    unsigned OddSrc = StartIndexes[1];
    if (EvenSrc == 0 && OddSrc == NumElts) {
      if (Subtarget.is64Bit())
        return DAG.getNode(RISCVISD::PZIP, DL, VT, V1, V2);
      EVT HalfVT = VT.getHalfNumVectorElementsVT();
      V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
      V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
      return DAG.getNode(RISCVISD::PWZIP, DL, VT, V1, V2);
    }
    if (EvenSrc == NumElts && OddSrc == 0) {
      if (Subtarget.is64Bit())
        return DAG.getNode(RISCVISD::PZIP, DL, VT, V2, V1);
      EVT HalfVT = VT.getHalfNumVectorElementsVT();
      V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
      V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
      return DAG.getNode(RISCVISD::PWZIP, DL, VT, V2, V1);
    }
  }
 
  return SDValue();
}
 
// Match a deinterleave shuffle that forms a P-extension packed unzip:
//   <a0, a2, ..., b0, b2, ...> -> unzip*p
//   <a1, a3, ..., b1, b3, ...> -> unzip*hp
static SDValue lowerVECTOR_SHUFFLEAsPUnzip(ShuffleVectorSDNode *SVN,
                                           SelectionDAG &DAG, bool IsRV64) {
  MVT VT = SVN->getSimpleValueType(0);
  if (!IsRV64 || (VT != MVT::v8i8 && VT != MVT::v4i16))
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  ArrayRef<int> Mask = SVN->getMask();
 
  unsigned Index = 0;
  if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index))
    return SDValue();
 
  unsigned Opc = Index == 0 ? RISCVISD::PUNZIPE : RISCVISD::PUNZIPO;
  return DAG.getNode(Opc, DL, VT, V1, V2);
}
 
// Match a slide by one element with a scalar inserted at either end:
//   <b0, a0, ..., aN-2> -> slide1up
//   <a1, ..., aN-1, b0> -> slide1down
static SDValue lowerVECTOR_SHUFFLEAsPSlide1(ShuffleVectorSDNode *SVN,
                                            const RISCVSubtarget &Subtarget,
                                            SelectionDAG &DAG) {
  MVT VT = SVN->getSimpleValueType(0);
  if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v2i16 &&
      VT != MVT::v4i16 && VT != MVT::v2i32)
    return SDValue();
  if (!Subtarget.is64Bit() && VT == MVT::v2i32)
    return SDValue();
 
  unsigned NumElts = VT.getVectorNumElements();
  ArrayRef<int> Mask = SVN->getMask();
  unsigned ActiveElts = NumElts;
  if (Subtarget.is64Bit() &&
      all_of(Mask.drop_front(NumElts / 2), [](int M) { return M < 0; }))
    ActiveElts /= 2;
 
  ArrayRef<int> ActiveMask = Mask.take_front(ActiveElts);
  bool SlideUp = ActiveMask[0] == (int)NumElts &&
                 all_of(enumerate(ActiveMask.drop_front()), [](const auto &M) {
                   return M.value() == (int)M.index();
                 });
  bool SlideDown = ActiveMask.back() == (int)NumElts &&
                   all_of(enumerate(ActiveMask.drop_back()), [](const auto &M) {
                     return M.value() == (int)M.index() + 1;
                   });
  if (!SlideUp && !SlideDown)
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  unsigned EltBits = VT.getScalarSizeInBits();
  unsigned SlideBits = ActiveElts * EltBits;
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Scalar = DAG.getExtractVectorElt(DL, XLenVT, V2, 0);
 
  // A two-element slide is exactly a packed pair operation.
  if (ActiveElts == 2) {
    SDValue ScalarVec = DAG.getBitcast(VT, Scalar);
    return DAG.getNode(SlideUp ? RISCVISD::PPAIRE : RISCVISD::PPAIROE, DL, VT,
                       SlideUp ? ScalarVec : V1, SlideUp ? V1 : ScalarVec);
  }
 
  // Lower a full-register slide to the funnel shift instruction that
  // implements it.
  if (SlideBits == Subtarget.getXLen()) {
    SDValue Bits = DAG.getBitcast(XLenVT, V1);
    SDValue Shamt = DAG.getConstant(EltBits, DL, XLenVT);
    if (SlideUp) {
      Scalar = DAG.getNode(ISD::SHL, DL, XLenVT, Scalar,
                           DAG.getConstant(SlideBits - EltBits, DL, XLenVT));
      Bits = DAG.getNode(ISD::FSHL, DL, XLenVT, Bits, Scalar, Shamt);
    } else {
      Bits = DAG.getNode(ISD::FSHR, DL, XLenVT, Scalar, Bits, Shamt);
    }
    return DAG.getBitcast(VT, Bits);
  }
 
  // A 32-bit packed slide on RV64 is carried in the low word of a GPR. Expand
  // it using XLEN operations or packed pair instructions.
  if (Subtarget.is64Bit()) {
    assert(SlideBits == 32 && "Unexpected RV64 packed slide width");
    SDValue Shamt = DAG.getConstant(EltBits, DL, MVT::i64);
    SDValue Bits = DAG.getBitcast(MVT::i64, V1);
    if (SlideUp) {
      Scalar = DAG.getNode(ISD::SHL, DL, MVT::i64, Scalar,
                           DAG.getConstant(64 - EltBits, DL, MVT::i64));
      Bits = DAG.getNode(ISD::FSHL, DL, MVT::i64, Bits, Scalar, Shamt);
    } else {
      SDValue Pair = DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i32,
                                 DAG.getBitcast(MVT::v2i32, V1),
                                 DAG.getBitcast(MVT::v2i32, Scalar));
      Bits = DAG.getNode(ISD::SRL, DL, MVT::i64, DAG.getBitcast(MVT::i64, Pair),
                         Shamt);
    }
    return DAG.getBitcast(VT, Bits);
  }
 
  // RV32 represents 64-bit packed values as two GPRs. Form each shifted half
  // directly so the slide uses two XLEN funnel shifts.
  assert(SlideBits == 64 && "Unexpected RV32 packed slide width");
  auto [LoVec, HiVec] = DAG.SplitVector(V1, DL);
  MVT HalfVT = LoVec.getSimpleValueType();
  SDValue Lo = DAG.getBitcast(MVT::i32, LoVec);
  SDValue Hi = DAG.getBitcast(MVT::i32, HiVec);
  SDValue Shamt = DAG.getConstant(EltBits, DL, MVT::i32);
  SDValue NewLo;
  SDValue NewHi;
  if (SlideUp) {
    SDValue Insert = DAG.getNode(ISD::SHL, DL, MVT::i32, Scalar,
                                 DAG.getConstant(32 - EltBits, DL, MVT::i32));
    NewLo = DAG.getNode(ISD::FSHL, DL, MVT::i32, Lo, Insert, Shamt);
    NewHi = DAG.getNode(ISD::FSHL, DL, MVT::i32, Hi, Lo, Shamt);
  } else {
    NewLo = DAG.getNode(ISD::FSHR, DL, MVT::i32, Hi, Lo, Shamt);
    NewHi = DAG.getNode(ISD::FSHR, DL, MVT::i32, Scalar, Hi, Shamt);
  }
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, DAG.getBitcast(HalfVT, NewLo),
                     DAG.getBitcast(HalfVT, NewHi));
}
 
// Match the packed zero-extend shuffle mask <0, N, 2, N+2, ...>: even result
// lanes keep operand 0's even lanes and odd result lanes come from operand 1.
// The odd lanes may select any element of operand 1, which is looser than a
// strict pair-even mask; DAGCombiner::XformToShuffleWithZero forms exactly this
// from a packed zero-extend `and`, leaving each zeroed lane at its own
// position, and lowerVECTOR_SHUFFLEAsPPair then forms the PPAIRE. Undef lanes
// always match.
static bool isPackedZExtShuffleMask(ArrayRef<int> Mask) {
  unsigned NumElts = Mask.size();
  if (NumElts % 2 != 0)
    return false;
  for (unsigned I = 0; I != NumElts / 2; ++I)
    if ((Mask[2 * I] >= 0 && Mask[2 * I] != (int)(2 * I)) ||
        (Mask[2 * I + 1] >= 0 && Mask[2 * I + 1] < (int)NumElts))
      return false;
  return true;
}
 
// Match a legalized deinterleave shuffle on two RV32 vector halves and lower
// it to an RV32 P narrowing shift on the concatenated source.
static SDValue
lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(ShuffleVectorSDNode *SVN,
                                         const RISCVSubtarget &Subtarget,
                                         SelectionDAG &DAG) {
  MVT VT = SVN->getSimpleValueType(0);
  if (VT != MVT::v4i8)
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  unsigned NumElts = VT.getVectorNumElements();
 
  SDValue Src = foldConcatVector(V1, V2);
  if (!Src) {
    MVT SrcVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
    Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, V1, V2);
  }
 
  // The source vector should be twice the size.
  if (Src.getValueType().getVectorNumElements() != 2 * NumElts)
    return SDValue();
 
  unsigned Index = 0;
  if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(SVN->getMask(), 2, Index))
    return SDValue();
 
  unsigned EltBits = VT.getVectorElementType().getSizeInBits();
  return DAG.getNode(RISCVISD::PNSRL, DL, VT, Src,
                     DAG.getConstant(Index * EltBits, DL, MVT::i32));
}
 
// Match a strided-interleave shuffle that forms a P-extension packed pair:
//   <a0, b0, a2, b2, ...> -> ppaire.*
//   <a0, b1, a2, b3, ...> -> ppaireo.*
//   <a1, b0, a3, b2, ...> -> ppairoe.*
//   <a1, b1, a3, b3, ...> -> ppairo.*
static SDValue lowerVECTOR_SHUFFLEAsPPair(ShuffleVectorSDNode *SVN,
                                          SelectionDAG &DAG) {
  MVT VT = SVN->getSimpleValueType(0);
  if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v2i16 &&
      VT != MVT::v4i16)
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  unsigned NumElts = VT.getVectorNumElements();
  ArrayRef<int> Mask = SVN->getMask();
 
  // A splat operand's lanes are all equal, so a lane selecting from it matches
  // any of its positions. This covers the zero operand XformToShuffleWithZero
  // forms for a packed zero-extend, which keeps each zeroed lane at its own
  // position rather than the strided one.
  bool V1IsSplat = DAG.isSplatValue(V1);
  bool V2IsSplat = DAG.isSplatValue(V2);
 
  // Walk the mask once, tracking the operand feeding the destination's even
  // lanes (index 0) and the operand feeding its odd lanes (index 1) — either
  // may turn out to be V1 or V2 — along with whether each pulls the even or
  // odd element out of its pair. All even (resp. odd) lanes must agree on
  // both the operand and the parity used; a splat operand's lanes are all
  // equal so it never constrains the parity.
  SDValue Src[2];
  std::optional<bool> Parity[2];
  for (unsigned I = 0; I != NumElts; ++I) {
    int M = Mask[I];
    if (M < 0)
      continue;
    unsigned Lane = I % 2;
    bool FromV1 = (unsigned)M < NumElts;
    SDValue Cand = FromV1 ? V1 : V2;
    unsigned Local = (unsigned)M % NumElts;
    if (!Src[Lane])
      Src[Lane] = Cand;
    else if (Src[Lane] != Cand)
      return SDValue();
 
    // Splats don't constrain parity.
    if (FromV1 ? V1IsSplat : V2IsSplat)
      continue;
 
    // The index must be from the even/odd element of its pair.
    if (Local / 2 != I / 2)
      return SDValue();
 
    bool P = Local % 2;
    if (!Parity[Lane])
      Parity[Lane] = P;
    else if (*Parity[Lane] != P)
      return SDValue();
  }
 
  // Make sure we have a source for both lanes.
  if (!Src[0] || !Src[1])
    return SDValue();
 
  bool EvenIsOdd = Parity[0].value_or(false);
  bool OddIsOdd = Parity[1].value_or(false);
  unsigned Opc;
  if (!EvenIsOdd && !OddIsOdd)
    Opc = RISCVISD::PPAIRE;
  else if (EvenIsOdd && OddIsOdd)
    Opc = RISCVISD::PPAIRO;
  else if (!EvenIsOdd && OddIsOdd)
    Opc = RISCVISD::PPAIREO;
  else
    Opc = RISCVISD::PPAIROE;
 
  return DAG.getNode(Opc, DL, VT, Src[0], Src[1]);
}
 
SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDValue V1 = Op.getOperand(0);
  SDValue V2 = Op.getOperand(1);
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  unsigned NumElts = VT.getVectorNumElements();
  ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
 
  // Select RVP-specific packed shuffles before falling back to the generic
  // fixed/scalable-vector lowering below.
  if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
    ArrayRef<int> Mask = SVN->getMask();
 
    // Select an element reverse shuffle to VECTOR_REVERSE. The tablegen
    // patterns select rev8/rev16/ppairoe.* from VECTOR_REVERSE.
    // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
    // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
    auto IsLowReverse = [&](unsigned L) {
      return V2.isUndef() &&
             ShuffleVectorInst::isReverseMask(Mask.take_front(L), L) &&
             all_of(Mask.drop_front(L), [](int M) { return M < 0; });
    };
    if (IsLowReverse(NumElts))
      return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
    if (Subtarget.is64Bit() && VT == MVT::v4i16 && IsLowReverse(/*L=*/2))
      return DAG.getNode(RISCVISD::PPAIROE_H, DL, VT, V1, V1);
    // Widened: reversing sends the low-half lanes to the top half, so shift
    // them back down by half the register. Only the 64-bit packed types are
    // legal here, so the register is XLen (i64).
    if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
        IsLowReverse(NumElts / 2)) {
      SDValue Rev = DAG.getBitcast(
          MVT::i64, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1));
      SDValue Srl =
          DAG.getNode(ISD::SRL, DL, MVT::i64, Rev,
                      DAG.getConstant(VT.getSizeInBits() / 2, DL, MVT::i64));
      return DAG.getBitcast(VT, Srl);
    }
 
    if (SDValue V = lowerVECTOR_SHUFFLEAsPSlide1(SVN, Subtarget, DAG))
      return V;
    if (SDValue V = lowerVECTOR_SHUFFLEAsPUnzip(SVN, DAG, Subtarget.is64Bit()))
      return V;
    if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, Subtarget, DAG))
      return V;
    if (SDValue V =
            lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(SVN, Subtarget, DAG))
      return V;
    if (SDValue V = lowerVECTOR_SHUFFLEAsPPair(SVN, DAG))
      return V;
    return SDValue();
  }
 
  if (VT.getVectorElementType() == MVT::i1) {
    // Lower to a vror.vi of a larger element type if possible before we promote
    // i1s to i8s.
    if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
      return V;
    if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
      return V;
 
    // Promote i1 shuffle to i8 shuffle.
    MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
    V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
    V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
                      : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
    SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
    return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
                        ISD::SETNE);
  }
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
  // Store the return value in a single variable instead of structured bindings
  // so that we can pass it to GetSlide below, which cannot capture structured
  // bindings until C++20.
  auto TrueMaskVL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
  auto [TrueMask, VL] = TrueMaskVL;
 
  if (SVN->isSplat()) {
    const int Lane = SVN->getSplatIndex();
    if (Lane >= 0) {
      MVT SVT = VT.getVectorElementType();
 
      // Turn splatted vector load into a strided load with an X0 stride.
      SDValue V = V1;
      // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
      // with undef.
      // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
      int Offset = Lane;
      if (V.getOpcode() == ISD::CONCAT_VECTORS) {
        int OpElements =
            V.getOperand(0).getSimpleValueType().getVectorNumElements();
        V = V.getOperand(Offset / OpElements);
        Offset %= OpElements;
      }
 
      // We need to ensure the load isn't atomic or volatile.
      if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
        auto *Ld = cast<LoadSDNode>(V);
        Offset *= SVT.getStoreSize();
        SDValue NewAddr = DAG.getMemBasePlusOffset(
            Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
 
        // If this is SEW=64 on RV32, use a strided load with a stride of x0.
        if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
          SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
          SDValue IntID =
              DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
          SDValue Ops[] = {Ld->getChain(),
                           IntID,
                           DAG.getUNDEF(ContainerVT),
                           NewAddr,
                           DAG.getRegister(RISCV::X0, XLenVT),
                           VL};
          SDValue NewLoad = DAG.getMemIntrinsicNode(
              ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
              DAG.getMachineFunction().getMachineMemOperand(
                  Ld->getMemOperand(), Offset, SVT.getStoreSize()));
          DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
          return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
        }
 
        MVT SplatVT = ContainerVT;
 
        // f16 with zvfhmin and bf16 need to use an integer scalar load.
        if (SVT == MVT::bf16 ||
            (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
          SVT = MVT::i16;
          SplatVT = ContainerVT.changeVectorElementType(SVT);
        }
 
        // Otherwise use a scalar load and splat. This will give the best
        // opportunity to fold a splat into the operation. ISel can turn it into
        // the x0 strided load if we aren't able to fold away the select.
        if (SVT.isFloatingPoint())
          V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
                          Ld->getPointerInfo().getWithOffset(Offset),
                          Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
        else
          V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
                             Ld->getPointerInfo().getWithOffset(Offset), SVT,
                             Ld->getBaseAlign(),
                             Ld->getMemOperand()->getFlags());
        DAG.makeEquivalentMemoryOrdering(Ld, V);
 
        unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
                                                 : RISCVISD::VMV_V_X_VL;
        SDValue Splat =
            DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
        Splat = DAG.getBitcast(ContainerVT, Splat);
        return convertFromScalableVector(VT, Splat, DAG, Subtarget);
      }
 
      V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
      assert(Lane < (int)NumElts && "Unexpected lane!");
      SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
                                   V1, DAG.getConstant(Lane, DL, XLenVT),
                                   DAG.getUNDEF(ContainerVT), TrueMask, VL);
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
  }
 
  // For exact VLEN m2 or greater, try to split to m1 operations if we
  // can split cleanly.
  if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
    return V;
 
  ArrayRef<int> Mask = SVN->getMask();
 
  if (SDValue V =
          lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
    return V;
 
  if (SDValue V =
          lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
    return V;
 
  // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
  // available.
  if (Subtarget.hasStdExtZvkb())
    if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
      return V;
 
  if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef() &&
      NumElts != 2)
    return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
 
  // Try using vunzip{e,o} from Zvzip to lower deinterleave2.
  unsigned Index = 0;
  if (Subtarget.hasStdExtZvzip() &&
      ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index) &&
      1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
    bool UsesBothSources =
        1 < count_if(Mask,
                     [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
        1 < count_if(Mask,
                     [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
 
    unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
    if (isLegalVTForZvzipDeinterleavedOperand(VT, Subtarget)) {
      MVT NewVT = VT.getDoubleNumVectorElementsVT();
      if (isTypeLegal(NewVT)) {
        SDValue Op;
        if (V2.isUndef()) {
          Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
        } else if (auto VLEN = Subtarget.getRealVLen();
                   VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
          Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
        } else if (SDValue Src = foldConcatVector(V1, V2)) {
          Op = DAG.getExtractSubvector(DL, NewVT, Src, 0);
        }
        if (Op)
          return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
      }
    }
 
    MVT HalfVT = VT.getHalfNumVectorElementsVT();
    if (UsesBothSources &&
        isLegalVTForZvzipDeinterleavedOperand(HalfVT, Subtarget) &&
        V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
        V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
      SDValue Lo = lowerZvzipVUNZIP(Opc, V1, DL, DAG, Subtarget);
      SDValue Hi = lowerZvzipVUNZIP(Opc, V2, DL, DAG, Subtarget);
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
  }
 
  // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
  // use shift and truncate to perform the shuffle.
  // TODO: For Factor=6, we can perform the first step of the deinterleave via
  // shift-and-trunc reducing total cost for everything except an mf8 result.
  // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
  // to do the entire operation.
  if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
    const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
    assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
    for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
      unsigned Index = 0;
      if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
          1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
        if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
          return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
        if (1 < count_if(Mask,
                         [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
            1 < count_if(Mask, [&Mask](int Idx) {
              return Idx >= (int)Mask.size();
            })) {
          // Narrow each source and concatenate them.
          // FIXME: For small LMUL it is better to concatenate first.
          MVT EltVT = VT.getVectorElementType();
          auto EltCnt = VT.getVectorElementCount();
          MVT SubVT =
              MVT::getVectorVT(EltVT, EltCnt.divideCoefficientBy(Factor));
 
          SDValue Lo =
              getDeinterleaveShiftAndTrunc(DL, SubVT, V1, Factor, Index, DAG);
          SDValue Hi =
              getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG);
 
          SDValue Concat =
              DAG.getNode(ISD::CONCAT_VECTORS, DL,
                          SubVT.getDoubleNumVectorElementsVT(), Lo, Hi);
          if (Factor == 2)
            return Concat;
 
          SDValue Vec = DAG.getUNDEF(VT);
          return DAG.getInsertSubvector(DL, Vec, Concat, 0);
        }
      }
    }
  }
 
  if (SDValue V =
          lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
    return V;
 
  // Detect an interleave shuffle and lower to
  // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
  int EvenSrc, OddSrc;
  if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
      !(NumElts == 2 &&
        ShuffleVectorInst::isSingleSourceMask(Mask, Mask.size()))) {
    // Extract the halves of the vectors.
    MVT HalfVT = VT.getHalfNumVectorElementsVT();
 
    // Recognize if one half is actually undef; the matching above will
    // otherwise reuse the even stream for the undef one.  This improves
    // spread(2) shuffles.
    bool LaneIsUndef[2] = { true, true};
    for (const auto &[Idx, M] : enumerate(Mask))
      LaneIsUndef[Idx % 2] &= (M == -1);
 
    int Size = Mask.size();
    SDValue EvenV, OddV;
    if (LaneIsUndef[0]) {
      EvenV = DAG.getUNDEF(HalfVT);
    } else {
      assert(EvenSrc >= 0 && "Undef source?");
      EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
      EvenV = DAG.getExtractSubvector(DL, HalfVT, EvenV, EvenSrc % Size);
    }
 
    if (LaneIsUndef[1]) {
      OddV = DAG.getUNDEF(HalfVT);
    } else {
      assert(OddSrc >= 0 && "Undef source?");
      OddV = (OddSrc / Size) == 0 ? V1 : V2;
      OddV = DAG.getExtractSubvector(DL, HalfVT, OddV, OddSrc % Size);
    }
 
    // Prefer vzip if available.
    // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
    if (Subtarget.hasStdExtZvzip() &&
        isLegalVTForZvzipInterleavedOperand(VT, Subtarget))
      return lowerZvzipVZIP(EvenV, OddV, DL, DAG, Subtarget);
    return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
  }
 
  // Recognize a pattern which can handled via a pair of vslideup/vslidedown
  // instructions (in any combination) with masking on the second instruction.
  // Also handles masked slides into an identity source, and single slides
  // without masking.  Avoid matching bit rotates (which are not also element
  // rotates) as slide pairs.  This is a performance heuristic, not a
  // functional check.
  std::array<std::pair<int, int>, 2> SrcInfo;
  unsigned RotateAmt;
  MVT RotateVT;
  if (::isMaskedSlidePair(Mask, SrcInfo) &&
      (isElementRotate(SrcInfo, NumElts) ||
       !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
    SDValue Sources[2];
    auto GetSourceFor = [&](const std::pair<int, int> &Info) {
      int SrcIdx = Info.first;
      assert(SrcIdx == 0 || SrcIdx == 1);
      SDValue &Src = Sources[SrcIdx];
      if (!Src) {
        SDValue SrcV = SrcIdx == 0 ? V1 : V2;
        Src = convertToScalableVector(ContainerVT, SrcV, DAG, Subtarget);
      }
      return Src;
    };
    auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
                        SDValue Passthru) {
      auto [TrueMask, VL] = TrueMaskVL;
      SDValue SrcV = GetSourceFor(Src);
      int SlideAmt = Src.second;
      if (SlideAmt == 0) {
        // Should never be second operation
        assert(Mask == TrueMask);
        return SrcV;
      }
      if (SlideAmt < 0)
        return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
                             DAG.getConstant(-SlideAmt, DL, XLenVT), Mask, VL,
                             RISCVVType::TAIL_AGNOSTIC);
      return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
                         DAG.getConstant(SlideAmt, DL, XLenVT), Mask, VL,
                         RISCVVType::TAIL_AGNOSTIC);
    };
 
    if (SrcInfo[1].first == -1) {
      SDValue Res = DAG.getUNDEF(ContainerVT);
      Res = GetSlide(SrcInfo[0], TrueMask, Res);
      return convertFromScalableVector(VT, Res, DAG, Subtarget);
    }
 
    if (Subtarget.hasStdExtZvzip()) {
      bool TryWiden = false;
      unsigned Factor;
      if (isPairEven(SrcInfo, Mask, Factor)) {
        if (Factor == 1) {
          SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
          SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
          return lowerZvzipVPAIR(RISCVISD::VPAIRE_VL, Src1, Src2, DL, DAG,
                                 Subtarget);
        }
        TryWiden = true;
      }
      if (isPairOdd(SrcInfo, Mask, Factor)) {
        if (Factor == 1) {
          SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
          SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
          return lowerZvzipVPAIR(RISCVISD::VPAIRO_VL, Src1, Src2, DL, DAG,
                                 Subtarget);
        }
        TryWiden = true;
      }
      // If we found a widening oppurtunity which would let us form a
      // pair-even or pair-odd, use the generic code to widen the shuffle
      // and recurse through this logic.
      if (TryWiden)
        if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
          return V;
    }
 
    // Build the mask.  Note that vslideup unconditionally preserves elements
    // below the slide amount in the destination, and thus those elements are
    // undefined in the mask.  If the mask ends up all true (or undef), it
    // will be folded away by general logic.
    SmallVector<SDValue> MaskVals;
    for (const auto &[Idx, M] : enumerate(Mask)) {
      if (M < 0 ||
          (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
        MaskVals.push_back(DAG.getUNDEF(XLenVT));
        continue;
      }
      int Src = M >= (int)NumElts;
      int Diff = (int)Idx - (M % NumElts);
      bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
      assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
             "Must match exactly one of the two slides");
      MaskVals.push_back(DAG.getConstant(C, DL, XLenVT));
    }
    assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
    MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
    SDValue SelectMask = convertToScalableVector(
        ContainerVT.changeVectorElementType(MVT::i1),
        DAG.getBuildVector(MaskVT, DL, MaskVals), DAG, Subtarget);
 
    SDValue Res = DAG.getUNDEF(ContainerVT);
    Res = GetSlide(SrcInfo[0], TrueMask, Res);
    Res = GetSlide(SrcInfo[1], SelectMask, Res);
    return convertFromScalableVector(VT, Res, DAG, Subtarget);
  }
 
  // Handle any remaining single source shuffles
  assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
  if (V2.isUndef()) {
    // We might be able to express the shuffle as a bitrotate. But even if we
    // don't have Zvkb and have to expand, the expanded sequence of approx. 2
    // shifts and a vor will have a higher throughput than a vrgather.
    if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
      return V;
 
    if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
      return V;
 
    // Match a spread(4,8) which can be done via extend and shift.  Spread(2)
    // is fully covered in interleave(2) above, so it is ignored here.
    if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
      unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
      assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
      for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
        unsigned Index;
        if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
          MVT NarrowVT =
              MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
          SDValue Src = DAG.getExtractSubvector(DL, NarrowVT, V1, 0);
          return getWideningSpread(Src, Factor, Index, DL, DAG);
        }
      }
    }
 
    // If only a prefix of the source elements influence a prefix of the
    // destination elements, try to see if we can reduce the required LMUL
    unsigned MinVLen = Subtarget.getRealMinVLen();
    unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
    if (NumElts > MinVLMAX) {
      unsigned MaxIdx = 0;
      for (auto [I, M] : enumerate(Mask)) {
        if (M == -1)
          continue;
        MaxIdx = std::max({(unsigned)I, (unsigned)M, MaxIdx});
      }
      unsigned NewNumElts =
          std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
      if (NewNumElts != NumElts) {
        MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
        V1 = DAG.getExtractSubvector(DL, NewVT, V1, 0);
        SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
                                           Mask.take_front(NewNumElts));
        return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
      }
    }
 
    // Before hitting generic lowering fallbacks, try to widen the mask
    // to a wider SEW.
    if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
      return V;
 
    // Can we generate a vcompress instead of a vrgather?  These scale better
    // at high LMUL, at the cost of not being able to fold a following select
    // into them.  The mask constants are also smaller than the index vector
    // constants, and thus easier to materialize.
    if (isCompressMask(Mask)) {
      SmallVector<SDValue> MaskVals(NumElts,
                                    DAG.getConstant(false, DL, XLenVT));
      for (auto Idx : Mask) {
        if (Idx == -1)
          break;
        assert(Idx >= 0 && (unsigned)Idx < NumElts);
        MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
      }
      MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
      SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
      return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
                         DAG.getUNDEF(VT));
    }
 
    if (VT.getScalarSizeInBits() == 8 &&
        any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
      // On such a vector we're unable to use i8 as the index type.
      // FIXME: We could promote the index to i16 and use vrgatherei16, but that
      // may involve vector splitting if we're already at LMUL=8, or our
      // user-supplied maximum fixed-length LMUL.
      return SDValue();
    }
 
    // Base case for the two operand recursion below - handle the worst case
    // single source shuffle.
    unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
    MVT IndexVT = VT.changeTypeToInteger();
    // Since we can't introduce illegal index types at this stage, use i16 and
    // vrgatherei16 if the corresponding index type for plain vrgather is greater
    // than XLenVT.
    if (IndexVT.getScalarType().bitsGT(XLenVT)) {
      GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
      IndexVT = IndexVT.changeVectorElementType(MVT::i16);
    }
 
    // If the mask allows, we can do all the index computation in 16 bits.  This
    // requires less work and less register pressure at high LMUL, and creates
    // smaller constants which may be cheaper to materialize.
    if (IndexVT.getScalarType().bitsGT(MVT::i16) && isUInt<16>(NumElts - 1) &&
        (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
      GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
      IndexVT = IndexVT.changeVectorElementType(MVT::i16);
    }
 
    MVT IndexContainerVT =
      ContainerVT.changeVectorElementType(IndexVT.getScalarType());
 
    V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
    SmallVector<SDValue> GatherIndicesLHS;
    for (int MaskIndex : Mask) {
      bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
      GatherIndicesLHS.push_back(IsLHSIndex
                                 ? DAG.getConstant(MaskIndex, DL, XLenVT)
                                 : DAG.getUNDEF(XLenVT));
    }
    SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
    LHSIndices =
        convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
    // At m1 and less, there's no point trying any of the high LMUL splitting
    // techniques.  TODO: Should we reconsider this for DLEN < VLEN?
    if (NumElts <= MinVLMAX) {
      SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
                                   DAG.getUNDEF(ContainerVT), TrueMask, VL);
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
    EVT SubIndexVT = M1VT.changeVectorElementType(IndexVT.getScalarType());
    auto [InnerTrueMask, InnerVL] =
        getDefaultScalableVLOps(M1VT, DL, DAG, Subtarget);
    int N =
        ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
    assert(isPowerOf2_32(N) && N <= 8);
 
    // If we have a locally repeating mask, then we can reuse the first
    // register in the index register group for all registers within the
    // source register group.  TODO: This generalizes to m2, and m4.
    if (isLocalRepeatingShuffle(Mask, MinVLMAX)) {
      SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
      SDValue Gather = DAG.getUNDEF(ContainerVT);
      for (int i = 0; i < N; i++) {
        unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
        SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, SubIdx);
        SDValue SubVec =
            DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
                        DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
        Gather = DAG.getInsertSubvector(DL, Gather, SubVec, SubIdx);
      }
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    // If we have a shuffle which only uses the first register in our source
    // register group, and repeats the same index across all spans, we can
    // use a single vrgather (and possibly some register moves).
    // TODO: This can be generalized for m2 or m4, or for any shuffle for
    // which we can do a linear number of shuffles to form an m1 which
    // contains all the output elements.
    if (isLowSourceShuffle(Mask, MinVLMAX) &&
        isSpanSplatShuffle(Mask, MinVLMAX)) {
      SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
      SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
      SDValue SubVec = DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
                                   DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
      SDValue Gather = DAG.getUNDEF(ContainerVT);
      for (int i = 0; i < N; i++)
        Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
                                        M1VT.getVectorMinNumElements() * i);
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    // If we have a shuffle which only uses the first register in our
    // source register group, we can do a linear number of m1 vrgathers
    // reusing the same source register (but with different indices)
    // TODO: This can be generalized for m2 or m4, or for any shuffle
    // for which we can do a vslidedown followed by this expansion.
    if (isLowSourceShuffle(Mask, MinVLMAX)) {
      SDValue SlideAmt =
          DAG.getElementCount(DL, XLenVT, M1VT.getVectorElementCount());
      SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
      SDValue Gather = DAG.getUNDEF(ContainerVT);
      for (int i = 0; i < N; i++) {
        if (i != 0)
          LHSIndices = getVSlidedown(DAG, Subtarget, DL, IndexContainerVT,
                                     DAG.getUNDEF(IndexContainerVT), LHSIndices,
                                     SlideAmt, TrueMask, VL);
        SDValue SubIndex =
            DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
        SDValue SubVec =
            DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
                        DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
        Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
                                        M1VT.getVectorMinNumElements() * i);
      }
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    // Fallback to generic vrgather if we can't find anything better.
    // On many machines, this will be O(LMUL^2)
    SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
                                 DAG.getUNDEF(ContainerVT), TrueMask, VL);
    return convertFromScalableVector(VT, Gather, DAG, Subtarget);
  }
 
  // As a backup, shuffles can be lowered via a vrgather instruction, possibly
  // merged with a second vrgather.
  SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
 
  // Now construct the mask that will be used by the blended vrgather operation.
  // Construct the appropriate indices into each vector.
  for (int MaskIndex : Mask) {
    bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
    ShuffleMaskLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
                             ? MaskIndex : -1);
    ShuffleMaskRHS.push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
  }
 
  // If the mask indices are disjoint between the two sources, we can lower it
  // as a vselect + a single source vrgather.vv. Don't do this if we think the
  // operands may end up being lowered to something cheaper than a vrgather.vv.
  if (!DAG.isSplatValue(V2) && !DAG.isSplatValue(V1) &&
      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS) &&
      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS) &&
      !ShuffleVectorInst::isIdentityMask(ShuffleMaskLHS, NumElts) &&
      !ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts))
    if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
      return V;
 
  // Before hitting generic lowering fallbacks, try to widen the mask
  // to a wider SEW.
  if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
    return V;
 
  // Try to pick a profitable operand order.
  bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
  SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts);
 
  // Recursively invoke lowering for each operand if we had two
  // independent single source shuffles, and then combine the result via a
  // vselect.  Note that the vselect will likely be folded back into the
  // second permute (vrgather, or other) by the post-isel combine.
  V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
  V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
 
  SmallVector<SDValue> MaskVals;
  for (int MaskIndex : Mask) {
    bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
    MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
  }
 
  assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
  SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
 
  if (SwapOps)
    return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
  return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
}
 
bool RISCVTargetLowering::isVectorClearMaskLegal(ArrayRef<int> M,
                                                 EVT VT) const {
  // Enable DAGCombiner::XformToShuffleWithZero to rewrite a packed zero-extend
  // `and` into shuffle(src, zero, ...), which lowerVECTOR_SHUFFLE turns into
  // RISCVISD::PPAIRE. Accept the packed byte/halfword views that lowering
  // handles; the 32-bit views are illegal on RV64 but reachable before type
  // legalization widens them to the legal 64-bit view.
  if (!Subtarget.hasStdExtP() || !VT.isSimple())
    return false;
  MVT SVT = VT.getSimpleVT();
  return (SVT == MVT::v4i8 || SVT == MVT::v8i8 || SVT == MVT::v4i16) &&
         isPackedZExtShuffleMask(M);
}
 
bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
  // Only support legal VTs for other shuffles for now.
  if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
    return false;
 
  // Support splats for any type. These should type legalize well.
  if (ShuffleVectorSDNode::isSplatMask(M))
    return true;
 
  const unsigned NumElts = M.size();
  MVT SVT = VT.getSimpleVT();
 
  // Not for i1 vectors.
  if (SVT.getScalarType() == MVT::i1)
    return false;
 
  std::array<std::pair<int, int>, 2> SrcInfo;
  int Dummy1, Dummy2;
  return ShuffleVectorInst::isReverseMask(M, NumElts) ||
         (::isMaskedSlidePair(M, SrcInfo) &&
          isElementRotate(SrcInfo, NumElts)) ||
         isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
}
 
// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
// the exponent.
SDValue
RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
                                                SelectionDAG &DAG) const {
  MVT VT = Op.getSimpleValueType();
  unsigned EltSize = VT.getScalarSizeInBits();
  SDValue Src = Op.getOperand(0);
  SDLoc DL(Op);
  MVT ContainerVT = VT;
 
  // We choose FP type that can represent the value if possible. Otherwise, we
  // use rounding to zero conversion for correct exponent of the result.
  // TODO: Use f16 for i8 when possible?
  MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
  if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
    FloatEltVT = MVT::f32;
  MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
 
  // Legal types should have been checked in the RISCVTargetLowering
  // constructor.
  // TODO: Splitting may make sense in some cases.
  assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
         "Expected legal float type!");
 
  // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
  // The trailing zero count is equal to log2 of this single bit value.
  if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
    SDValue Neg = DAG.getNegative(Src, DL, VT);
    Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
  }
 
  // We have a legal FP type, convert to it.
  SDValue FloatVal;
  if (FloatVT.bitsGT(VT)) {
    FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
  } else {
    // Use RTZ to avoid rounding influencing exponent of FloatVal.
    if (VT.isFixedLengthVector()) {
      ContainerVT = getContainerForFixedLengthVector(VT);
      Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
    }
    auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
    SDValue RTZRM =
        DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
    MVT ContainerFloatVT =
        MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
    FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
                           Src, Mask, RTZRM, VL);
    if (VT.isFixedLengthVector())
      FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
  }
  // Bitcast to integer and shift the exponent to the LSB.
  EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
  SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
  unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
 
  // Restore back to original type. Truncation after SRL is to generate vnsrl.
  SDValue Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
                            DAG.getConstant(ShiftAmt, DL, IntVT));
  if (IntVT.bitsLT(VT))
    Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
  else if (IntVT.bitsGT(VT))
    Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
 
  // The exponent contains log2 of the value in biased form.
  unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
  // For trailing zeros, we just need to subtract the bias.
  if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
    return DAG.getNode(ISD::SUB, DL, VT, Exp,
                       DAG.getConstant(ExponentBias, DL, VT));
 
  // For leading zeros, we need to remove the bias and convert from log2 to
  // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
  unsigned Adjust = ExponentBias + (EltSize - 1);
  SDValue Res =
      DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
 
  // The above result with zero input equals to Adjust which is greater than
  // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
  if (Op.getOpcode() == ISD::CTLZ)
    Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
 
  return Res;
}
 
SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Source = Op->getOperand(0);
  MVT SrcVT = Source.getSimpleValueType();
  SDValue Mask = Op->getOperand(1);
  SDValue EVL = Op->getOperand(2);
 
  if (SrcVT.isFixedLengthVector()) {
    MVT ContainerVT = getContainerForFixedLengthVector(SrcVT);
    Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
    Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
                                   Subtarget);
    SrcVT = ContainerVT;
  }
 
  // Convert to boolean vector.
  if (SrcVT.getScalarType() != MVT::i1) {
    SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
    SrcVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorElementCount());
    Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
                         {Source, AllZero, DAG.getCondCode(ISD::SETNE),
                          DAG.getUNDEF(SrcVT), Mask, EVL});
  }
 
  SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
  if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
    // In this case, we can interpret poison as -1, so nothing to do further.
    return Res;
 
  // Convert -1 to VL.
  SDValue SetCC =
      DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
  Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
  return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
}
 
// While RVV has alignment restrictions, we should always be able to load as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
// the load is already correctly-aligned, it returns SDValue().
SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
                                                    SelectionDAG &DAG) const {
  auto *Load = cast<LoadSDNode>(Op);
  assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Load->getMemoryVT(),
                                     *Load->getMemOperand()))
    return SDValue();
 
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV load type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
  SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
                          Load->getPointerInfo(), Load->getBaseAlign(),
                          Load->getMemOperand()->getFlags());
  return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
}
 
// While RVV has alignment restrictions, we should always be able to store as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
// returns SDValue() if the store is already correctly aligned.
SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
                                                     SelectionDAG &DAG) const {
  auto *Store = cast<StoreSDNode>(Op);
  assert(Store && Store->getValue().getValueType().isVector() &&
         "Expected vector store");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Store->getMemoryVT(),
                                     *Store->getMemOperand()))
    return SDValue();
 
  SDLoc DL(Op);
  SDValue StoredVal = Store->getValue();
  MVT VT = StoredVal.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV store type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
  StoredVal = DAG.getBitcast(NewVT, StoredVal);
  return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
                      Store->getPointerInfo(), Store->getBaseAlign(),
                      Store->getMemOperand()->getFlags());
}
 
// While RVV has alignment restrictions, we should always be able to load as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
// the load is already correctly-aligned, it returns SDValue().
SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
                                                   SelectionDAG &DAG) const {
  auto *Load = cast<VPLoadSDNode>(Op);
  assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Load->getMemoryVT(),
                                     *Load->getMemOperand()))
    return SDValue();
 
  SDValue Mask = Load->getMask();
 
  // FIXME: Handled masked loads somehow.
  if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
    return SDValue();
 
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV load type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
 
  SDValue VL = Load->getVectorLength();
  VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
                   DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
 
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
  SDValue L = DAG.getLoadVP(NewVT, DL, Load->getChain(), Load->getBasePtr(),
                            DAG.getAllOnesConstant(DL, MaskVT), VL,
                            Load->getPointerInfo(), Load->getBaseAlign(),
                            Load->getMemOperand()->getFlags(), AAMDNodes());
  return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
}
 
// While RVV has alignment restrictions, we should always be able to store as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
// It returns SDValue() if the store is already correctly aligned.
SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
                                                    SelectionDAG &DAG) const {
  auto *Store = cast<VPStoreSDNode>(Op);
  assert(Store && Store->getValue().getValueType().isVector() &&
         "Expected vector store");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Store->getMemoryVT(),
                                     *Store->getMemOperand()))
    return SDValue();
 
  SDValue Mask = Store->getMask();
 
  // FIXME: Handled masked stores somehow.
  if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
    return SDValue();
 
  SDLoc DL(Op);
  SDValue StoredVal = Store->getValue();
  MVT VT = StoredVal.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV store type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
 
  SDValue VL = Store->getVectorLength();
  VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
                   DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
 
  StoredVal = DAG.getBitcast(NewVT, StoredVal);
 
  LocationSize Size = LocationSize::precise(NewVT.getStoreSize());
  MachineFunction &MF = DAG.getMachineFunction();
  MachineMemOperand *MMO = MF.getMachineMemOperand(
      Store->getPointerInfo(), Store->getMemOperand()->getFlags(), Size,
      Store->getBaseAlign());
 
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
  return DAG.getStoreVP(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
                        DAG.getPOISON(Store->getBasePtr().getValueType()),
                        DAG.getAllOnesConstant(DL, MaskVT), VL, NewVT, MMO,
                        ISD::UNINDEXED);
}
 
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget) {
  assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
 
  int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
 
  // All simm32 constants should be handled by isel.
  // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
  // this check redundant, but small immediates are common so this check
  // should have better compile time.
  if (isInt<32>(Imm))
    return Op;
 
  // We only need to cost the immediate, if constant pool lowering is enabled.
  if (!Subtarget.useConstantPoolForLargeInts())
    return Op;
 
  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget);
  if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
    return Op;
 
  // Optimizations below are disabled for opt size. If we're optimizing for
  // size, use a constant pool.
  if (DAG.shouldOptForSize())
    return SDValue();
 
  // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
  // that if it will avoid a constant pool.
  // It will require an extra temporary register though.
  // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
  // low and high 32 bits are the same and bit 31 and 63 are set.
  unsigned ShiftAmt, AddOpc;
  RISCVMatInt::InstSeq SeqLo =
      RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc);
  if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
    return Op;
 
  return SDValue();
}
 
SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
                                             SelectionDAG &DAG) const {
  MVT VT = Op.getSimpleValueType();
  const APFloat &Imm = cast<ConstantFPSDNode>(Op)->getValueAPF();
 
  // Can this constant be selected by a Zfa FLI instruction?
  bool Negate = false;
  int Index = getLegalZfaFPImm(Imm, VT);
 
  // If the constant is negative, try negating.
  if (Index < 0 && Imm.isNegative()) {
    Index = getLegalZfaFPImm(-Imm, VT);
    Negate = true;
  }
 
  // If we couldn't find a FLI lowering, fall back to generic code.
  if (Index < 0)
    return SDValue();
 
  // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
  SDLoc DL(Op);
  SDValue Const =
      DAG.getNode(RISCVISD::FLI, DL, VT,
                  DAG.getTargetConstant(Index, DL, Subtarget.getXLenVT()));
  if (!Negate)
    return Const;
 
  return DAG.getNode(ISD::FNEG, DL, VT, Const);
}
 
static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
                             SelectionDAG &DAG) {
 
  unsigned IsData = Op.getConstantOperandVal(4);
 
  // mips-p8700  we support data prefetch for now.
  if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
    return Op.getOperand(0);
  return Op;
}
 
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  SDLoc dl(Op);
  AtomicOrdering FenceOrdering =
      static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
  SyncScope::ID FenceSSID =
      static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
 
  if (Subtarget.hasStdExtZtso()) {
    // The only fence that needs an instruction is a sequentially-consistent
    // cross-thread fence.
    if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
        FenceSSID == SyncScope::System)
      return Op;
 
    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
    return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
  }
 
  // singlethread fences only synchronize with signal handlers on the same
  // thread and thus only need to preserve instruction order, not actually
  // enforce memory ordering.
  if (FenceSSID == SyncScope::SingleThread)
    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
    return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
 
  return Op;
}
 
SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
                                             SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned Check = Op.getConstantOperandVal(1);
  unsigned TDCMask = 0;
  if (Check & fcSNan)
    TDCMask |= RISCV::FPMASK_Signaling_NaN;
  if (Check & fcQNan)
    TDCMask |= RISCV::FPMASK_Quiet_NaN;
  if (Check & fcPosInf)
    TDCMask |= RISCV::FPMASK_Positive_Infinity;
  if (Check & fcNegInf)
    TDCMask |= RISCV::FPMASK_Negative_Infinity;
  if (Check & fcPosNormal)
    TDCMask |= RISCV::FPMASK_Positive_Normal;
  if (Check & fcNegNormal)
    TDCMask |= RISCV::FPMASK_Negative_Normal;
  if (Check & fcPosSubnormal)
    TDCMask |= RISCV::FPMASK_Positive_Subnormal;
  if (Check & fcNegSubnormal)
    TDCMask |= RISCV::FPMASK_Negative_Subnormal;
  if (Check & fcPosZero)
    TDCMask |= RISCV::FPMASK_Positive_Zero;
  if (Check & fcNegZero)
    TDCMask |= RISCV::FPMASK_Negative_Zero;
 
  bool IsOneBitMask = isPowerOf2_32(TDCMask);
 
  SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
 
  if (VT.isVector()) {
    SDValue Op0 = Op.getOperand(0);
    MVT VT0 = Op.getOperand(0).getSimpleValueType();
 
    if (VT.isScalableVector()) {
      MVT DstVT = VT0.changeVectorElementTypeToInteger();
      auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
      SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
                                    VL, Op->getFlags());
      if (IsOneBitMask)
        return DAG.getSetCC(DL, VT, FPCLASS,
                            DAG.getConstant(TDCMask, DL, DstVT),
                            ISD::CondCode::SETEQ);
      SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
                                DAG.getConstant(TDCMask, DL, DstVT));
      return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
                          ISD::SETNE);
    }
 
    MVT ContainerVT0 = getContainerForFixedLengthVector(VT0);
    MVT ContainerVT = getContainerForFixedLengthVector(VT);
    MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
    auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
    Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget);
 
    SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
                                  Mask, VL, Op->getFlags());
 
    TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
                           DAG.getUNDEF(ContainerDstVT), TDCMaskV, VL);
    if (IsOneBitMask) {
      SDValue VMSEQ =
          DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
                      {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
                       DAG.getUNDEF(ContainerVT), Mask, VL});
      return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
    }
    SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
                              TDCMaskV, DAG.getUNDEF(ContainerDstVT), Mask, VL);
 
    SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
    SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
                            DAG.getUNDEF(ContainerDstVT), SplatZero, VL);
 
    SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
                                {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
                                 DAG.getUNDEF(ContainerVT), Mask, VL});
    return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
  }
 
  SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
  SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
  SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
                             ISD::CondCode::SETNE);
  return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
}
 
// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
// operations propagate nans.
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
 
  SDValue X = Op.getOperand(0);
  SDValue Y = Op.getOperand(1);
 
  if (!VT.isVector()) {
    MVT XLenVT = Subtarget.getXLenVT();
 
    // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
    // ensures that when one input is a nan, the other will also be a nan
    // allowing the nan to propagate. If both inputs are nan, this will swap the
    // inputs which is harmless.
 
    SDValue NewY = Y;
    if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
      SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
      NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
    }
 
    SDValue NewX = X;
    if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
      SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
      NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
    }
 
    unsigned Opc =
        Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
    return DAG.getNode(Opc, DL, VT, NewX, NewY);
  }
 
  // Check no NaNs before converting to fixed vector scalable.
  bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
  bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
    Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  SDValue NewY = Y;
  if (!XIsNeverNan) {
    SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                                    {X, X, DAG.getCondCode(ISD::SETOEQ),
                                     DAG.getUNDEF(ContainerVT), Mask, VL});
    NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
                       DAG.getUNDEF(ContainerVT), VL);
  }
 
  SDValue NewX = X;
  if (!YIsNeverNan) {
    SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                                    {Y, Y, DAG.getCondCode(ISD::SETOEQ),
                                     DAG.getUNDEF(ContainerVT), Mask, VL});
    NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
                       DAG.getUNDEF(ContainerVT), VL);
  }
 
  unsigned Opc =
      Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
  SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
                            DAG.getUNDEF(ContainerVT), Mask, VL);
  if (VT.isFixedLengthVector())
    Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
  return Res;
}
 
static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
                               const RISCVSubtarget &Subtarget) {
  bool IsFABS = Op.getOpcode() == ISD::FABS;
  assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
         "Wrong opcode for lowering FABS or FNEG.");
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
 
  SDLoc DL(Op);
  SDValue Fmv =
      DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op.getOperand(0));
 
  APInt Mask = IsFABS ? APInt::getSignedMaxValue(16) : APInt::getSignMask(16);
  Mask = Mask.sext(Subtarget.getXLen());
 
  unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
  SDValue Logic =
      DAG.getNode(LogicOpc, DL, XLenVT, Fmv, DAG.getConstant(Mask, DL, XLenVT));
  return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, Logic);
}
 
static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
 
  SDValue Mag = Op.getOperand(0);
  SDValue Sign = Op.getOperand(1);
 
  SDLoc DL(Op);
 
  // Get sign bit into an integer value.
  unsigned SignSize = Sign.getValueSizeInBits();
  SDValue SignAsInt = [&]() {
    if (SignSize == Subtarget.getXLen())
      return DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
    switch (SignSize) {
    case 16:
      return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
    case 32:
      return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
    case 64: {
      assert(XLenVT == MVT::i32 && "Unexpected type");
      // Copy the upper word to integer.
      SignSize = 32;
      return DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
          .getValue(1);
    }
    default:
      llvm_unreachable("Unexpected sign size");
    }
  }();
 
  // Get the signbit at the right position for MagAsInt.
  if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
    SignAsInt = DAG.getNode(ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, XLenVT,
                            SignAsInt,
                            DAG.getConstant(std::abs(ShiftAmount), DL, XLenVT));
 
  // Mask the sign bit and any bits above it. The extra bits will be dropped
  // when we convert back to FP.
  SDValue SignMask = DAG.getConstant(
      APInt::getSignMask(16).sext(Subtarget.getXLen()), DL, XLenVT);
  SDValue SignBit = DAG.getNode(ISD::AND, DL, XLenVT, SignAsInt, SignMask);
 
  // Transform Mag value to integer, and clear the sign bit.
  SDValue MagAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Mag);
  SDValue ClearSignMask = DAG.getConstant(
      APInt::getSignedMaxValue(16).sext(Subtarget.getXLen()), DL, XLenVT);
  SDValue ClearedSign =
      DAG.getNode(ISD::AND, DL, XLenVT, MagAsInt, ClearSignMask);
 
  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit,
                                   SDNodeFlags::Disjoint);
 
  return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
}
 
/// Get a RISC-V target specified VL op for a given SDNode.
static unsigned getRISCVVLOp(SDValue Op) {
#define OP_CASE(NODE)                                                          \
  case ISD::NODE:                                                              \
    return RISCVISD::NODE##_VL;
#define VP_CASE(NODE)                                                          \
  case ISD::VP_##NODE:                                                         \
    return RISCVISD::NODE##_VL;
  // clang-format off
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
  OP_CASE(ADD)
  OP_CASE(SUB)
  OP_CASE(MUL)
  OP_CASE(MULHS)
  OP_CASE(MULHU)
  OP_CASE(SDIV)
  OP_CASE(SREM)
  OP_CASE(UDIV)
  OP_CASE(UREM)
  OP_CASE(SHL)
  OP_CASE(SRA)
  OP_CASE(SRL)
  OP_CASE(ROTL)
  OP_CASE(ROTR)
  OP_CASE(BSWAP)
  OP_CASE(CTTZ)
  OP_CASE(CTLZ)
  OP_CASE(CTPOP)
  OP_CASE(BITREVERSE)
  OP_CASE(CLMUL)
  OP_CASE(CLMULH)
  OP_CASE(SADDSAT)
  OP_CASE(UADDSAT)
  OP_CASE(SSUBSAT)
  OP_CASE(USUBSAT)
  OP_CASE(AVGFLOORS)
  OP_CASE(AVGFLOORU)
  OP_CASE(AVGCEILS)
  OP_CASE(AVGCEILU)
  OP_CASE(FADD)
  OP_CASE(FSUB)
  OP_CASE(FMUL)
  OP_CASE(FDIV)
  OP_CASE(FNEG)
  OP_CASE(FABS)
  OP_CASE(FCOPYSIGN)
  OP_CASE(FSQRT)
  OP_CASE(SMIN)
  OP_CASE(SMAX)
  OP_CASE(UMIN)
  OP_CASE(UMAX)
  OP_CASE(ABDS)
  OP_CASE(ABDU)
  OP_CASE(STRICT_FADD)
  OP_CASE(STRICT_FSUB)
  OP_CASE(STRICT_FMUL)
  OP_CASE(STRICT_FDIV)
  OP_CASE(STRICT_FSQRT)
  VP_CASE(SDIV)       // VP_SDIV
  VP_CASE(SREM)       // VP_SREM
  VP_CASE(UDIV)       // VP_UDIV
  VP_CASE(UREM)       // VP_UREM
  case ISD::CTLZ_ZERO_POISON:
    return RISCVISD::CTLZ_VL;
  case ISD::CTTZ_ZERO_POISON:
    return RISCVISD::CTTZ_VL;
  case ISD::FMA:
    return RISCVISD::VFMADD_VL;
  case ISD::STRICT_FMA:
    return RISCVISD::STRICT_VFMADD_VL;
  case ISD::AND:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return RISCVISD::VMAND_VL;
    return RISCVISD::AND_VL;
  case ISD::OR:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return RISCVISD::VMOR_VL;
    return RISCVISD::OR_VL;
  case ISD::XOR:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return RISCVISD::VMXOR_VL;
    return RISCVISD::XOR_VL;
  case ISD::ANY_EXTEND:
  case ISD::ZERO_EXTEND:
    return RISCVISD::VZEXT_VL;
  case ISD::SIGN_EXTEND:
    return RISCVISD::VSEXT_VL;
  case ISD::SETCC:
    return RISCVISD::SETCC_VL;
  case ISD::VSELECT:
    return RISCVISD::VMERGE_VL;
  case ISD::VP_MERGE:
    return RISCVISD::VMERGE_VL;
  case ISD::FMINNUM:
  case ISD::FMINIMUMNUM:
    return RISCVISD::VFMIN_VL;
  case ISD::FMAXNUM:
  case ISD::FMAXIMUMNUM:
    return RISCVISD::VFMAX_VL;
  case ISD::LRINT:
  case ISD::LLRINT:
    return RISCVISD::VFCVT_RM_X_F_VL;
  case ISD::MASKED_UDIV:
    return RISCVISD::UDIV_VL;
  case ISD::MASKED_UREM:
    return RISCVISD::UREM_VL;
  case ISD::MASKED_SDIV:
    return RISCVISD::SDIV_VL;
  case ISD::MASKED_SREM:
    return RISCVISD::SREM_VL;
  }
  // clang-format on
#undef OP_CASE
#undef VP_CASE
}
 
static bool isPromotedOpNeedingSplit(SDValue Op,
                                     const RISCVSubtarget &Subtarget,
                                     const TargetLowering &TLI) {
  MVT OpVT = Op.getSimpleValueType();
  if (!OpVT.isVector())
    return false;
  MVT EltVT = OpVT.getVectorElementType();
  if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
        !Subtarget.hasVInstructionsF16()) &&
      !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
        (!Subtarget.hasVInstructionsBF16() ||
         !llvm::is_contained(ZvfbfaOps, Op.getOpcode()))))
    return false;
  // Need to split when the same width f32 vector type isn't legal.
  return !TLI.isTypeLegal(
      MVT::getVectorVT(MVT::f32, OpVT.getVectorElementCount()));
}
 
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
  SDLoc DL(Op);
 
  SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
  SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
 
  for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
    if (!Op.getOperand(j).getValueType().isVector()) {
      LoOperands[j] = Op.getOperand(j);
      HiOperands[j] = Op.getOperand(j);
      continue;
    }
    std::tie(LoOperands[j], HiOperands[j]) =
        DAG.SplitVector(Op.getOperand(j), DL);
  }
 
  SDValue LoRes =
      DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
  SDValue HiRes =
      DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
 
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
}
 
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
                                      bool IsVP) {
  SDLoc DL(Op);
 
  if (IsVP) {
    auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
    auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
    auto [EVLLo, EVLHi] =
        DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
 
    SDValue ResLo =
        DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                    {Op.getOperand(0), Lo, MaskLo, EVLLo}, Op->getFlags());
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                       {ResLo, Hi, MaskHi, EVLHi}, Op->getFlags());
  }
 
  unsigned Opcode = Op.getOpcode();
  unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
 
  auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(OpNo), DL);
  if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
    SDValue ResLo = DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                                Op.getOperand(0), Lo, Op->getFlags());
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), ResLo, Hi,
                       Op->getFlags());
  }
 
  SDValue ResLo =
      DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Lo, Op->getFlags());
  SDValue ResHi =
      DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Hi, Op->getFlags());
  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
  return DAG.getNode(BaseOpc, DL, Op.getValueType(), ResLo, ResHi,
                     Op->getFlags());
}
 
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
 
  assert(Op->isStrictFPOpcode());
 
  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
 
  SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
  SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
 
  SDLoc DL(Op);
 
  SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
  SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
 
  for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
    if (!Op.getOperand(j).getValueType().isVector()) {
      LoOperands[j] = Op.getOperand(j);
      HiOperands[j] = Op.getOperand(j);
      continue;
    }
    std::tie(LoOperands[j], HiOperands[j]) =
        DAG.SplitVector(Op.getOperand(j), DL);
  }
 
  SDValue LoRes =
      DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
  HiOperands[0] = LoRes.getValue(1);
  SDValue HiRes =
      DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
 
  SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
                          LoRes.getValue(0), HiRes.getValue(0));
  return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
}
 
SDValue
RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
                                                   SelectionDAG &DAG) const {
  assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
         "Unexpected bfloat16 load lowering");
 
  SDLoc DL(Op);
  LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
  EVT MemVT = LD->getMemoryVT();
  SDValue Load = DAG.getExtLoad(
      ISD::ZEXTLOAD, DL, Subtarget.getXLenVT(), LD->getChain(),
      LD->getBasePtr(),
      EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
      LD->getMemOperand());
  // Using mask to make bf16 nan-boxing valid when we don't have flh
  // instruction. -65536 would be treat as a small number and thus it can be
  // directly used lui to get the constant.
  SDValue mask = DAG.getSignedConstant(-65536, DL, Subtarget.getXLenVT());
  SDValue OrSixteenOne =
      DAG.getNode(ISD::OR, DL, Load.getValueType(), {Load, mask});
  SDValue ConvertedResult =
      DAG.getNode(RISCVISD::NDS_FMV_BF16_X, DL, MVT::bf16, OrSixteenOne);
  return DAG.getMergeValues({ConvertedResult, Load.getValue(1)}, DL);
}
 
SDValue
RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
                                                    SelectionDAG &DAG) const {
  assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
         "Unexpected bfloat16 store lowering");
 
  StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
  SDLoc DL(Op);
  SDValue FMV = DAG.getNode(RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
                            Subtarget.getXLenVT(), ST->getValue());
  return DAG.getTruncStore(
      ST->getChain(), DL, FMV, ST->getBasePtr(),
      EVT::getIntegerVT(*DAG.getContext(), ST->getMemoryVT().getSizeInBits()),
      ST->getMemOperand());
}
 
static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget);
 
static SDValue lowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG,
                                              const RISCVSubtarget &Subtarget);
 
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                            SelectionDAG &DAG) const {
  switch (Op.getOpcode()) {
  default:
    reportFatalInternalError(
        "Unimplemented RISCVTargetLowering::LowerOperation Case");
  case ISD::PREFETCH:
    return LowerPREFETCH(Op, Subtarget, DAG);
  case ISD::ATOMIC_FENCE:
    return LowerATOMIC_FENCE(Op, DAG, Subtarget);
  case ISD::GlobalAddress:
    return lowerGlobalAddress(Op, DAG);
  case ISD::BlockAddress:
    return lowerBlockAddress(Op, DAG);
  case ISD::ConstantPool:
    return lowerConstantPool(Op, DAG);
  case ISD::JumpTable:
    return lowerJumpTable(Op, DAG);
  case ISD::GlobalTLSAddress:
    return lowerGlobalTLSAddress(Op, DAG);
  case ISD::Constant:
    return lowerConstant(Op, DAG, Subtarget);
  case ISD::ConstantFP:
    return lowerConstantFP(Op, DAG);
  case ISD::SELECT:
    return lowerSELECT(Op, DAG);
  case ISD::BRCOND:
    return lowerBRCOND(Op, DAG);
  case ISD::VASTART:
    return lowerVASTART(Op, DAG);
  case ISD::FRAMEADDR:
    return lowerFRAMEADDR(Op, DAG);
  case ISD::RETURNADDR:
    return lowerRETURNADDR(Op, DAG);
  case ISD::SHL_PARTS:
    return lowerShiftLeftParts(Op, DAG);
  case ISD::SRA_PARTS:
    return lowerShiftRightParts(Op, DAG, true);
  case ISD::SRL_PARTS:
    return lowerShiftRightParts(Op, DAG, false);
  case ISD::ROTL:
  case ISD::ROTR:
    if (Op.getValueType().isFixedLengthVector()) {
      assert(Subtarget.hasStdExtZvkb());
      return lowerToScalableOp(Op, DAG);
    }
    assert(Subtarget.hasVendorXTHeadBb() &&
           !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
           "Unexpected custom legalization");
    // XTHeadBb only supports rotate by constant.
    if (!isa<ConstantSDNode>(Op.getOperand(1)))
      return SDValue();
    return Op;
  case ISD::BITCAST: {
    SDLoc DL(Op);
    EVT VT = Op.getValueType();
    SDValue Op0 = Op.getOperand(0);
    EVT Op0VT = Op0.getValueType();
    MVT XLenVT = Subtarget.getXLenVT();
    if (Op0VT == MVT::i16 &&
        ((VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
         (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
      SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
      return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, NewOp0);
    }
    if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
        Subtarget.hasStdExtFOrZfinx()) {
      SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
      return DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
    }
    if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() &&
        Subtarget.hasStdExtDOrZdinx()) {
      SDValue Lo, Hi;
      std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32);
      return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
    }
 
    if (Subtarget.hasStdExtP() && VT.isSimple() && Op0VT.isSimple()) {
      if (VT.getSimpleVT() == Subtarget.getXLenVT() &&
          Subtarget.isPExtPackedType(Op0VT.getSimpleVT()))
        return Op;
      if (Op0VT.getSimpleVT() == Subtarget.getXLenVT() &&
          Subtarget.isPExtPackedType(VT.getSimpleVT()))
        return Op;
    }
 
    // Consider other scalar<->scalar casts as legal if the types are legal.
    // Otherwise expand them.
    if (!VT.isVector() && !Op0VT.isVector()) {
      if (isTypeLegal(VT) && isTypeLegal(Op0VT))
        return Op;
      return SDValue();
    }
 
    assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
           "Unexpected types");
 
    if (VT.isFixedLengthVector()) {
      // We can handle fixed length vector bitcasts with a simple replacement
      // in isel.
      if (Op0VT.isFixedLengthVector())
        return Op;
      // When bitcasting from scalar to fixed-length vector, insert the scalar
      // into a one-element vector of the result type, and perform a vector
      // bitcast.
      if (!Op0VT.isVector()) {
        EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
        if (!isTypeLegal(BVT))
          return SDValue();
        return DAG.getBitcast(
            VT, DAG.getInsertVectorElt(DL, DAG.getUNDEF(BVT), Op0, 0));
      }
      return SDValue();
    }
    // Custom-legalize bitcasts from fixed-length vector types to scalar types
    // thus: bitcast the vector to a one-element vector type whose element type
    // is the same as the result type, and extract the first element.
    if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
      EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
      if (!isTypeLegal(BVT))
        return SDValue();
      SDValue BVec = DAG.getBitcast(BVT, Op0);
      return DAG.getExtractVectorElt(DL, VT, BVec, 0);
    }
    return SDValue();
  }
  case ISD::INTRINSIC_WO_CHAIN:
    return LowerINTRINSIC_WO_CHAIN(Op, DAG);
  case ISD::INTRINSIC_W_CHAIN:
    return LowerINTRINSIC_W_CHAIN(Op, DAG);
  case ISD::INTRINSIC_VOID:
    return LowerINTRINSIC_VOID(Op, DAG);
  case ISD::IS_FPCLASS:
    return LowerIS_FPCLASS(Op, DAG);
  case ISD::BITREVERSE: {
    MVT VT = Op.getSimpleValueType();
    if (VT.isFixedLengthVector()) {
      assert(Subtarget.hasStdExtZvbb());
      return lowerToScalableOp(Op, DAG);
    }
    SDLoc DL(Op);
    assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
    assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
    // Expand bitreverse to a bswap(rev8) followed by brev8.
    SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
    return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
  }
  case ISD::TRUNCATE:
  case ISD::TRUNCATE_SSAT_S:
  case ISD::TRUNCATE_USAT_U:
    // Only custom-lower vector truncates
    if (!Op.getSimpleValueType().isVector())
      return Op;
    return lowerVectorTrunc(Op, DAG);
  case ISD::ANY_EXTEND:
  case ISD::ZERO_EXTEND:
    if (Op.getOperand(0).getValueType().isVector() &&
        Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
      return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
    if (Op.getValueType().isScalableVector())
      return Op;
    return lowerToScalableOp(Op, DAG);
  case ISD::SIGN_EXTEND:
    if (Op.getOperand(0).getValueType().isVector() &&
        Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
      return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
    if (Op.getValueType().isScalableVector())
      return Op;
    return lowerToScalableOp(Op, DAG);
  case ISD::SPLAT_VECTOR_PARTS:
    return lowerSPLAT_VECTOR_PARTS(Op, DAG);
  case ISD::INSERT_VECTOR_ELT:
    return lowerINSERT_VECTOR_ELT(Op, DAG);
  case ISD::EXTRACT_VECTOR_ELT:
    return lowerEXTRACT_VECTOR_ELT(Op, DAG);
  case ISD::SCALAR_TO_VECTOR: {
    MVT VT = Op.getSimpleValueType();
    SDLoc DL(Op);
    SDValue Scalar = Op.getOperand(0);
    if (VT.getVectorElementType() == MVT::i1) {
      MVT WideVT = VT.changeVectorElementType(MVT::i8);
      SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar);
      return DAG.getNode(ISD::TRUNCATE, DL, VT, V);
    }
    MVT ContainerVT = VT;
    if (VT.isFixedLengthVector())
      ContainerVT = getContainerForFixedLengthVector(VT);
    SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
 
    SDValue V;
    if (VT.isFloatingPoint()) {
      V = DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, ContainerVT,
                      DAG.getUNDEF(ContainerVT), Scalar, VL);
    } else {
      Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
      V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
                      DAG.getUNDEF(ContainerVT), Scalar, VL);
    }
    if (VT.isFixedLengthVector())
      V = convertFromScalableVector(VT, V, DAG, Subtarget);
    return V;
  }
  case ISD::VSCALE: {
    MVT XLenVT = Subtarget.getXLenVT();
    MVT VT = Op.getSimpleValueType();
    SDLoc DL(Op);
    SDValue Res = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
    // We define our scalable vector types for lmul=1 to use a 64 bit known
    // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
    // vscale as VLENB / 8.
    static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
    if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
      reportFatalInternalError("Support for VLEN==32 is incomplete.");
    // We assume VLENB is a multiple of 8. We manually choose the best shift
    // here because SimplifyDemandedBits isn't always able to simplify it.
    uint64_t Val = Op.getConstantOperandVal(0);
    if (isPowerOf2_64(Val)) {
      uint64_t Log2 = Log2_64(Val);
      if (Log2 < 3) {
        SDNodeFlags Flags;
        Flags.setExact(true);
        Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
                          DAG.getConstant(3 - Log2, DL, XLenVT), Flags);
      } else if (Log2 > 3) {
        Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res,
                          DAG.getConstant(Log2 - 3, DL, XLenVT));
      }
    } else if ((Val % 8) == 0) {
      // If the multiplier is a multiple of 8, scale it down to avoid needing
      // to shift the VLENB value.
      Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res,
                        DAG.getConstant(Val / 8, DL, XLenVT));
    } else {
      SDNodeFlags Flags;
      Flags.setExact(true);
      SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
                                   DAG.getConstant(3, DL, XLenVT), Flags);
      Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale,
                        DAG.getConstant(Val, DL, XLenVT));
    }
    return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
  }
  case ISD::FPOWI: {
    // Custom promote f16 powi with illegal i32 integer type on RV64. Once
    // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
    if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
        Op.getOperand(1).getValueType() == MVT::i32) {
      SDLoc DL(Op);
      SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
      SDValue Powi =
          DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
      return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
                         DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
    }
    return SDValue();
  }
  case ISD::FMAXIMUM:
  case ISD::FMINIMUM:
    if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
      return SplitVectorOp(Op, DAG);
    return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
  case ISD::FP_EXTEND:
  case ISD::FP_ROUND:
    return lowerVectorFPExtendOrRound(Op, DAG);
  case ISD::STRICT_FP_ROUND:
  case ISD::STRICT_FP_EXTEND:
    return lowerStrictFPExtendOrRoundLike(Op, DAG);
  case ISD::SINT_TO_FP:
  case ISD::UINT_TO_FP:
    // An i1-to-fp conversion is equivalent to selecting between 1.0/-1.0 and
    // 0.0 for unsigned/signed conversion, respectively. This avoids extending
    // the mask before converting it.
    if (Op.getValueType().isVector() &&
        Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1) {
      SDLoc DL(Op);
      EVT VT = Op.getValueType();
      SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
      double TrueVal = Op.getOpcode() == ISD::UINT_TO_FP ? 1.0 : -1.0;
      SDValue True = DAG.getConstantFP(TrueVal, DL, VT);
      return DAG.getNode(ISD::VSELECT, DL, VT, Op.getOperand(0), True, Zero);
    }
    // Fall back to zvfbfmin for bf16 case if source type is wider than 8 bits.
    if (SDValue Op1 = Op.getOperand(0);
        Op.getValueType().isVector() &&
        ((Op.getValueType().getScalarType() == MVT::f16 &&
          (Subtarget.hasVInstructionsF16Minimal() &&
           !Subtarget.hasVInstructionsF16())) ||
         (Op.getValueType().getScalarType() == MVT::bf16 &&
          (Subtarget.hasVInstructionsBF16Minimal() &&
           (!Subtarget.hasVInstructionsBF16() ||
            Op1.getValueType().getScalarSizeInBits() > 8))))) {
      MVT NVT =
          MVT::getVectorVT(MVT::f32, Op.getValueType().getVectorElementCount());
      if (!isTypeLegal(NVT))
        return SplitVectorOp(Op, DAG);
      // int -> f32
      SDLoc DL(Op);
      SDValue NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
      // f32 -> [b]f16
      return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
                         DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
    }
    [[fallthrough]];
  case ISD::FP_TO_SINT:
  case ISD::FP_TO_UINT:
    // Fall back to zvfbfmin for bf16 case if destination type is wider than 8
    // bits.
    if (SDValue Op1 = Op.getOperand(0);
        Op1.getValueType().isVector() &&
        ((Op1.getValueType().getScalarType() == MVT::f16 &&
          (Subtarget.hasVInstructionsF16Minimal() &&
           !Subtarget.hasVInstructionsF16())) ||
         (Op1.getValueType().getScalarType() == MVT::bf16 &&
          (Subtarget.hasVInstructionsBF16Minimal() &&
           (!Subtarget.hasVInstructionsBF16() ||
            Op.getValueType().getScalarSizeInBits() > 8))))) {
      MVT NVT = MVT::getVectorVT(MVT::f32,
                                 Op1.getValueType().getVectorElementCount());
      if (!isTypeLegal(NVT))
        return SplitVectorOp(Op, DAG);
      // [b]f16 -> f32
      SDLoc DL(Op);
      SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
      // f32 -> int
      return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), WidenVec);
    }
    [[fallthrough]];
  case ISD::STRICT_FP_TO_SINT:
  case ISD::STRICT_FP_TO_UINT:
  case ISD::STRICT_SINT_TO_FP:
  case ISD::STRICT_UINT_TO_FP: {
    // RVV can only do fp<->int conversions to types half/double the size as
    // the source. We custom-lower any conversions that do two hops into
    // sequences.
    MVT VT = Op.getSimpleValueType();
    if (VT.isScalarInteger())
      return lowerFP_TO_INT(Op, DAG, Subtarget);
    bool IsStrict = Op->isStrictFPOpcode();
    SDValue Src = Op.getOperand(0 + IsStrict);
    MVT SrcVT = Src.getSimpleValueType();
    if (SrcVT.isScalarInteger())
      return lowerINT_TO_FP(Op, DAG, Subtarget);
    if (!VT.isVector())
      return Op;
    SDLoc DL(Op);
    MVT EltVT = VT.getVectorElementType();
    MVT SrcEltVT = SrcVT.getVectorElementType();
    unsigned EltSize = EltVT.getSizeInBits();
    unsigned SrcEltSize = SrcEltVT.getSizeInBits();
    assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
           "Unexpected vector element types");
 
    bool IsInt2FP = SrcEltVT.isInteger();
    // Widening conversions
    if (EltSize > (2 * SrcEltSize)) {
      if (IsInt2FP) {
        // Do a regular integer sign/zero extension then convert to float.
        MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize / 2),
                                      VT.getVectorElementCount());
        unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP ||
                              Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
                                 ? ISD::ZERO_EXTEND
                                 : ISD::SIGN_EXTEND;
        SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
        if (IsStrict)
          return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(),
                             Op.getOperand(0), Ext);
        return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
      }
      // FP2Int
      assert((SrcEltVT == MVT::f16 || SrcEltVT == MVT::bf16) &&
             "Unexpected FP_TO_[US]INT lowering");
      // Do one doubling fp_extend then complete the operation by converting
      // to int.
      MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
      if (IsStrict) {
        auto [FExt, Chain] =
            DAG.getStrictFPExtendOrRound(Src, Op.getOperand(0), DL, InterimFVT);
        return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), Chain, FExt);
      }
      SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
      return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
    }
 
    // Narrowing conversions
    if (SrcEltSize > (2 * EltSize)) {
      if (IsInt2FP) {
        // One narrowing int_to_fp, then an fp_round.
        assert((EltVT == MVT::f16 || EltVT == MVT::bf16) &&
               "Unexpected [US]_TO_FP lowering");
        MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        if (IsStrict) {
          SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL,
                                       DAG.getVTList(InterimFVT, MVT::Other),
                                       Op.getOperand(0), Src);
          SDValue Chain = Int2FP.getValue(1);
          return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first;
        }
        SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
        return DAG.getFPExtendOrRound(Int2FP, DL, VT);
      }
      // FP2Int
      // One narrowing fp_to_int, then truncate the integer. If the float isn't
      // representable by the integer, the result is poison.
      MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize / 2),
                                    VT.getVectorElementCount());
      if (IsStrict) {
        SDValue FP2Int =
            DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(IVecVT, MVT::Other),
                        Op.getOperand(0), Src);
        SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
        return DAG.getMergeValues({Res, FP2Int.getValue(1)}, DL);
      }
      SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
      if (EltSize == 1)
        // The integer should be 0 or 1/-1, so compare the integer result to 0.
        return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, IVecVT), FP2Int,
                            ISD::SETNE);
      return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
    }
 
    // Scalable vectors can exit here. Patterns will handle equally-sized
    // conversions halving/doubling ones.
    if (!VT.isFixedLengthVector())
      return Op;
 
    // For fixed-length vectors we lower to a custom "VL" node.
    unsigned RVVOpc = 0;
    switch (Op.getOpcode()) {
    default:
      llvm_unreachable("Impossible opcode");
    case ISD::FP_TO_SINT:
      RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
      break;
    case ISD::FP_TO_UINT:
      RVVOpc = RISCVISD::VFCVT_RTZ_XU_F_VL;
      break;
    case ISD::SINT_TO_FP:
      RVVOpc = RISCVISD::SINT_TO_FP_VL;
      break;
    case ISD::UINT_TO_FP:
      RVVOpc = RISCVISD::UINT_TO_FP_VL;
      break;
    case ISD::STRICT_FP_TO_SINT:
      RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_X_F_VL;
      break;
    case ISD::STRICT_FP_TO_UINT:
      RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL;
      break;
    case ISD::STRICT_SINT_TO_FP:
      RVVOpc = RISCVISD::STRICT_SINT_TO_FP_VL;
      break;
    case ISD::STRICT_UINT_TO_FP:
      RVVOpc = RISCVISD::STRICT_UINT_TO_FP_VL;
      break;
    }
 
    MVT ContainerVT = getContainerForFixedLengthVector(VT);
    MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
    assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() &&
           "Expected same element count");
 
    auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
    if (IsStrict) {
      Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
                        Op.getOperand(0), Src, Mask, VL);
      SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget);
      return DAG.getMergeValues({SubVec, Src.getValue(1)}, DL);
    }
    Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
    return convertFromScalableVector(VT, Src, DAG, Subtarget);
  }
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
    return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
  case ISD::FP_TO_BF16: {
    // Custom lower to ensure the libcall return is passed in an FPR on hard
    // float ABIs.
    assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization");
    SDLoc DL(Op);
    MakeLibCallOptions CallOptions;
    RTLIB::Libcall LC =
        RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16);
    SDValue Res =
        makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
    if (Subtarget.is64Bit())
      return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
    return DAG.getBitcast(MVT::i32, Res);
  }
  case ISD::BF16_TO_FP: {
    assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization");
    MVT VT = Op.getSimpleValueType();
    SDLoc DL(Op);
    Op = DAG.getNode(
        ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0),
        DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL));
    SDValue Res = Subtarget.is64Bit()
                      ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op)
                      : DAG.getBitcast(MVT::f32, Op);
    // fp_extend if the target VT is bigger than f32.
    if (VT != MVT::f32)
      return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
    return Res;
  }
  case ISD::STRICT_FP_TO_FP16:
  case ISD::FP_TO_FP16: {
    // Custom lower to ensure the libcall return is passed in an FPR on hard
    // float ABIs.
    assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
    SDLoc DL(Op);
    MakeLibCallOptions CallOptions;
    bool IsStrict = Op->isStrictFPOpcode();
    SDValue Op0 = IsStrict ? Op.getOperand(1) : Op.getOperand(0);
    SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
    RTLIB::Libcall LC = RTLIB::getFPROUND(Op0.getValueType(), MVT::f16);
    SDValue Res;
    std::tie(Res, Chain) =
        makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions, DL, Chain);
    if (Subtarget.is64Bit())
      return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
    SDValue Result = DAG.getBitcast(MVT::i32, IsStrict ? Res.getValue(0) : Res);
    if (IsStrict)
      return DAG.getMergeValues({Result, Chain}, DL);
    return Result;
  }
  case ISD::STRICT_FP16_TO_FP:
  case ISD::FP16_TO_FP: {
    // Custom lower to ensure the libcall argument is passed in an FPR on hard
    // float ABIs.
    assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
    SDLoc DL(Op);
    MakeLibCallOptions CallOptions;
    bool IsStrict = Op->isStrictFPOpcode();
    SDValue Op0 = IsStrict ? Op.getOperand(1) : Op.getOperand(0);
    SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
    SDValue Arg = Subtarget.is64Bit()
                      ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op0)
                      : DAG.getBitcast(MVT::f32, Op0);
    SDValue Res;
    std::tie(Res, Chain) = makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
                                       CallOptions, DL, Chain);
    if (IsStrict)
      return DAG.getMergeValues({Res, Chain}, DL);
    return Res;
  }
  case ISD::FTRUNC:
  case ISD::FCEIL:
  case ISD::FFLOOR:
  case ISD::FNEARBYINT:
  case ISD::FRINT:
  case ISD::FROUND:
  case ISD::FROUNDEVEN:
    if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
      return SplitVectorOp(Op, DAG);
    return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
  case ISD::FCANONICALIZE: {
    MVT VT = Op.getSimpleValueType();
    assert(VT.isFixedLengthVector() && "Unexpected type");
    SDLoc DL(Op);
    MVT ContainerVT = getContainerForFixedLengthVector(VT);
    SDValue Src =
        convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
    SDValue Res = DAG.getNode(ISD::FCANONICALIZE, DL, ContainerVT, Src);
    return convertFromScalableVector(VT, Res, DAG, Subtarget);
  }
  case ISD::LRINT:
  case ISD::LLRINT:
  case ISD::LROUND:
  case ISD::LLROUND: {
    if (Op.getValueType().isVector())
      return lowerVectorXRINT_XROUND(Op, DAG, Subtarget);
    assert(Op.getOperand(0).getValueType() == MVT::f16 &&
           "Unexpected custom legalisation");
    SDLoc DL(Op);
    SDValue Ext = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Ext);
  }
  case ISD::STRICT_LRINT:
  case ISD::STRICT_LLRINT:
  case ISD::STRICT_LROUND:
  case ISD::STRICT_LLROUND: {
    assert(Op.getOperand(1).getValueType() == MVT::f16 &&
           "Unexpected custom legalisation");
    SDLoc DL(Op);
    SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                              {Op.getOperand(0), Op.getOperand(1)});
    return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
                       {Ext.getValue(1), Ext.getValue(0)});
  }
  case ISD::VECREDUCE_ADD:
  case ISD::VECREDUCE_UMAX:
  case ISD::VECREDUCE_SMAX:
  case ISD::VECREDUCE_UMIN:
  case ISD::VECREDUCE_SMIN:
    return lowerVECREDUCE(Op, DAG);
  case ISD::VECREDUCE_AND:
  case ISD::VECREDUCE_OR:
  case ISD::VECREDUCE_XOR:
    if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
      return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
    return lowerVECREDUCE(Op, DAG);
  case ISD::VECREDUCE_SEQ_FADD:
    if (isPromotedOpNeedingSplit(Op.getOperand(1), Subtarget, *this))
      return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
    return lowerFPVECREDUCE(Op, DAG);
  case ISD::VECREDUCE_FADD:
  case ISD::VECREDUCE_FMIN:
  case ISD::VECREDUCE_FMAX:
  case ISD::VECREDUCE_FMAXIMUM:
  case ISD::VECREDUCE_FMINIMUM:
    if (isPromotedOpNeedingSplit(Op.getOperand(0), Subtarget, *this))
      return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
    return lowerFPVECREDUCE(Op, DAG);
  case ISD::VP_REDUCE_ADD:
  case ISD::VP_REDUCE_UMAX:
  case ISD::VP_REDUCE_SMAX:
  case ISD::VP_REDUCE_UMIN:
  case ISD::VP_REDUCE_SMIN:
  case ISD::VP_REDUCE_FADD:
  case ISD::VP_REDUCE_SEQ_FADD:
  case ISD::VP_REDUCE_FMIN:
  case ISD::VP_REDUCE_FMAX:
  case ISD::VP_REDUCE_FMINIMUM:
  case ISD::VP_REDUCE_FMAXIMUM:
    if (isPromotedOpNeedingSplit(Op.getOperand(1), Subtarget, *this))
      return SplitVectorReductionOp(Op, DAG, /*IsVP*/ true);
    return lowerVPREDUCE(Op, DAG);
  case ISD::VP_REDUCE_AND:
  case ISD::VP_REDUCE_OR:
  case ISD::VP_REDUCE_XOR:
    if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
      return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
    return lowerVPREDUCE(Op, DAG);
  case ISD::VP_CTTZ_ELTS:
  case ISD::VP_CTTZ_ELTS_ZERO_POISON:
    return lowerVPCttzElements(Op, DAG);
  case ISD::UNDEF:
  case ISD::POISON: {
    MVT ContainerVT = getContainerForFixedLengthVector(Op.getSimpleValueType());
    SDValue Passthru = Op.getOpcode() == ISD::POISON
                           ? DAG.getPOISON(ContainerVT)
                           : DAG.getUNDEF(ContainerVT);
    return convertFromScalableVector(Op.getSimpleValueType(), Passthru, DAG,
                                     Subtarget);
  }
  case ISD::INSERT_SUBVECTOR:
    return lowerINSERT_SUBVECTOR(Op, DAG);
  case ISD::EXTRACT_SUBVECTOR:
    return lowerEXTRACT_SUBVECTOR(Op, DAG);
  case ISD::VECTOR_DEINTERLEAVE:
    return lowerVECTOR_DEINTERLEAVE(Op, DAG);
  case ISD::VECTOR_INTERLEAVE:
    return lowerVECTOR_INTERLEAVE(Op, DAG);
  case ISD::STEP_VECTOR:
    return lowerSTEP_VECTOR(Op, DAG);
  case ISD::VECTOR_REVERSE:
    return lowerVECTOR_REVERSE(Op, DAG);
  case ISD::VECTOR_SPLICE_LEFT:
  case ISD::VECTOR_SPLICE_RIGHT:
    return lowerVECTOR_SPLICE(Op, DAG);
  case ISD::BUILD_VECTOR: {
    MVT VT = Op.getSimpleValueType();
    MVT EltVT = VT.getVectorElementType();
    if (!Subtarget.is64Bit() && EltVT == MVT::i64)
      return lowerBuildVectorViaVID(Op, DAG, Subtarget);
    return lowerBUILD_VECTOR(Op, DAG, Subtarget);
  }
  case ISD::SPLAT_VECTOR: {
    MVT VT = Op.getSimpleValueType();
    MVT EltVT = VT.getVectorElementType();
    if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
        EltVT == MVT::bf16) {
      SDLoc DL(Op);
      SDValue Elt;
      if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
          (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
        Elt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(),
                          Op.getOperand(0));
      else
        Elt = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op.getOperand(0));
      MVT IVT = VT.changeVectorElementType(MVT::i16);
      return DAG.getNode(ISD::BITCAST, DL, VT,
                         DAG.getNode(ISD::SPLAT_VECTOR, DL, IVT, Elt));
    }
 
    if (EltVT == MVT::i1)
      return lowerVectorMaskSplat(Op, DAG);
    return SDValue();
  }
  case ISD::VECTOR_SHUFFLE:
    return lowerVECTOR_SHUFFLE(Op, DAG);
  case ISD::CONCAT_VECTORS: {
    // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
    // better than going through the stack, as the default expansion does.
    SDLoc DL(Op);
    MVT VT = Op.getSimpleValueType();
    MVT ContainerVT = VT;
    if (VT.isFixedLengthVector())
      ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
 
    // Recursively split concat_vectors with more than 2 operands:
    //
    // concat_vector op1, op2, op3, op4
    // ->
    // concat_vector (concat_vector op1, op2), (concat_vector op3, op4)
    //
    // This reduces the length of the chain of vslideups and allows us to
    // perform the vslideups at a smaller LMUL, limited to MF2.
    if (Op.getNumOperands() > 2 &&
        ContainerVT.bitsGE(RISCVTargetLowering::getM1VT(ContainerVT))) {
      MVT HalfVT = VT.getHalfNumVectorElementsVT();
      assert(isPowerOf2_32(Op.getNumOperands()));
      size_t HalfNumOps = Op.getNumOperands() / 2;
      SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
                               Op->ops().take_front(HalfNumOps));
      SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
                               Op->ops().drop_front(HalfNumOps));
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
 
    unsigned NumOpElts =
        Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
    SDValue Vec = DAG.getUNDEF(VT);
    for (const auto &OpIdx : enumerate(Op->ops())) {
      SDValue SubVec = OpIdx.value();
      // Don't insert undef subvectors.
      if (SubVec.isUndef())
        continue;
      Vec = DAG.getInsertSubvector(DL, Vec, SubVec, OpIdx.index() * NumOpElts);
    }
    return Vec;
  }
  case ISD::LOAD: {
    auto *Load = cast<LoadSDNode>(Op);
    EVT VT = Load->getValueType(0);
    if (VT == MVT::f64) {
      assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
             !Subtarget.is64Bit() && "Unexpected custom legalisation");
 
      // Replace a double precision load with two i32 loads and a BuildPairF64.
      SDLoc DL(Op);
      SDValue BasePtr = Load->getBasePtr();
      SDValue Chain = Load->getChain();
 
      SDValue Lo =
          DAG.getLoad(MVT::i32, DL, Chain, BasePtr, Load->getPointerInfo(),
                      Load->getBaseAlign(), Load->getMemOperand()->getFlags());
      BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(4));
      SDValue Hi = DAG.getLoad(
          MVT::i32, DL, Chain, BasePtr, Load->getPointerInfo().getWithOffset(4),
          Load->getBaseAlign(), Load->getMemOperand()->getFlags());
      Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
                          Hi.getValue(1));
 
      // For big-endian, swap the order of Lo and Hi.
      if (!Subtarget.isLittleEndian())
        std::swap(Lo, Hi);
 
      SDValue Pair = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
      return DAG.getMergeValues({Pair, Chain}, DL);
    }
 
    if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
        (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
      assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
 
      // Determine the half-size type
      MVT HalfVT;
      if (VT == MVT::v2i32)
        HalfVT = MVT::i32;
      else if (VT == MVT::v4i16)
        HalfVT = MVT::v2i16;
      else // VT == MVT::v8i8
        HalfVT = MVT::v4i8;
 
      SDLoc DL(Op);
      SDValue BasePtr = Load->getBasePtr();
      SDValue Chain = Load->getChain();
 
      // Create two loads for the lower and upper halves
      SDValue Lo =
          DAG.getLoad(HalfVT, DL, Chain, BasePtr, Load->getPointerInfo(),
                      Load->getBaseAlign(), Load->getMemOperand()->getFlags());
      unsigned HalfSize = HalfVT.getStoreSize();
      BasePtr =
          DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(HalfSize));
      SDValue Hi =
          DAG.getLoad(HalfVT, DL, Chain, BasePtr,
                      Load->getPointerInfo().getWithOffset(HalfSize),
                      Load->getBaseAlign(), Load->getMemOperand()->getFlags());
      Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
                          Hi.getValue(1));
 
      // Combine the two halves into the result vector
      SDValue Result;
      if (VT == MVT::v2i32) {
        // For v2i32, build vector from two i32 scalars
        Result = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
      } else {
        // For v4i16 and v8i8, use CONCAT_VECTORS
        Result = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
      }
 
      return DAG.getMergeValues({Result, Chain}, DL);
    }
 
    if (VT == MVT::bf16)
      return lowerXAndesBfHCvtBFloat16Load(Op, DAG);
 
    // Handle normal vector tuple load.
    if (VT.isRISCVVectorTuple()) {
      SDLoc DL(Op);
      MVT XLenVT = Subtarget.getXLenVT();
      unsigned NF = VT.getRISCVVectorTupleNumFields();
      unsigned Sz = VT.getSizeInBits().getKnownMinValue();
      unsigned NumElts = Sz / (NF * 8);
      int Log2LMUL = Log2_64(NumElts) - 3;
 
      auto Flag = SDNodeFlags();
      Flag.setNoUnsignedWrap(true);
      SDValue Ret = DAG.getUNDEF(VT);
      SDValue BasePtr = Load->getBasePtr();
      SDValue VROffset = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
      VROffset =
          DAG.getNode(ISD::SHL, DL, XLenVT, VROffset,
                      DAG.getConstant(std::max(Log2LMUL, 0), DL, XLenVT));
      SmallVector<SDValue, 8> OutChains;
 
      // Load NF vector registers and combine them to a vector tuple.
      for (unsigned i = 0; i < NF; ++i) {
        SDValue LoadVal = DAG.getLoad(
            MVT::getScalableVectorVT(MVT::i8, NumElts), DL, Load->getChain(),
            BasePtr, MachinePointerInfo(Load->getAddressSpace()), Align(8));
        OutChains.push_back(LoadVal.getValue(1));
        Ret = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VT, Ret, LoadVal,
                          DAG.getTargetConstant(i, DL, MVT::i32));
        BasePtr = DAG.getNode(ISD::ADD, DL, XLenVT, BasePtr, VROffset, Flag);
      }
      return DAG.getMergeValues(
          {Ret, DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains)}, DL);
    }
 
    if (auto V = expandUnalignedRVVLoad(Op, DAG))
      return V;
    if (Op.getValueType().isFixedLengthVector())
      return lowerFixedLengthVectorLoadToRVV(Op, DAG);
    return Op;
  }
  case ISD::STORE: {
    auto *Store = cast<StoreSDNode>(Op);
    SDValue StoredVal = Store->getValue();
    EVT VT = StoredVal.getValueType();
 
    if (VT == MVT::f64) {
      assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
             !Subtarget.is64Bit() && "Unexpected custom legalisation");
 
      // Replace a double precision store with a SplitF64 and i32 stores.
      SDValue DL(Op);
      SDValue BasePtr = Store->getBasePtr();
      SDValue Chain = Store->getChain();
      SDValue Split = DAG.getNode(RISCVISD::SplitF64, DL,
                                  DAG.getVTList(MVT::i32, MVT::i32), StoredVal);
 
      SDValue Lo = Split.getValue(0);
      SDValue Hi = Split.getValue(1);
 
      // For big-endian, swap the order of Lo and Hi before storing.
      if (!Subtarget.isLittleEndian())
        std::swap(Lo, Hi);
 
      SDValue LoStore = DAG.getStore(
          Chain, DL, Lo, BasePtr, Store->getPointerInfo(),
          Store->getBaseAlign(), Store->getMemOperand()->getFlags());
      BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(4));
      SDValue HiStore = DAG.getStore(
          Chain, DL, Hi, BasePtr, Store->getPointerInfo().getWithOffset(4),
          Store->getBaseAlign(), Store->getMemOperand()->getFlags());
      return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoStore, HiStore);
    }
    if (VT == MVT::i64) {
      assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
             "Unexpected custom legalisation");
      if (Store->isTruncatingStore())
        return SDValue();
 
      if (Store->getAlign() < Subtarget.getZilsdAlign())
        return SDValue();
 
      SDLoc DL(Op);
      SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, StoredVal,
                               DAG.getTargetConstant(0, DL, MVT::i32));
      SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, StoredVal,
                               DAG.getTargetConstant(1, DL, MVT::i32));
 
      return DAG.getMemIntrinsicNode(
          RISCVISD::SD_RV32, DL, DAG.getVTList(MVT::Other),
          {Store->getChain(), Lo, Hi, Store->getBasePtr()}, MVT::i64,
          Store->getMemOperand());
    }
 
    if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
        (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
      assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
 
      auto *Store = cast<StoreSDNode>(Op);
      SDValue Val = Store->getValue();
 
      // Determine the half-size type
      MVT HalfVT;
      if (VT == MVT::v2i32)
        HalfVT = MVT::i32;
      else if (VT == MVT::v4i16)
        HalfVT = MVT::v2i16;
      else // VT == MVT::v8i8
        HalfVT = MVT::v4i8;
 
      SDLoc DL(Op);
      SDValue BasePtr = Store->getBasePtr();
      SDValue Chain = Store->getChain();
 
      // Extract the two halves from the vector
      SDValue Lo, Hi;
      if (VT == MVT::v2i32) {
        // For v2i32, extract two i32 scalars
        Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Val,
                         DAG.getVectorIdxConstant(0, DL));
        Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Val,
                         DAG.getVectorIdxConstant(1, DL));
      } else {
        // For v4i16 and v8i8, extract two vector halves
        Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Val,
                         DAG.getVectorIdxConstant(0, DL));
        unsigned HalfNumElts = HalfVT.getVectorNumElements();
        Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Val,
                         DAG.getVectorIdxConstant(HalfNumElts, DL));
      }
 
      // Create two stores for the lower and upper halves
      SDValue LoStore = DAG.getStore(
          Chain, DL, Lo, BasePtr, Store->getPointerInfo(),
          Store->getBaseAlign(), Store->getMemOperand()->getFlags());
      unsigned HalfSize = HalfVT.getStoreSize();
      BasePtr =
          DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(HalfSize));
      SDValue HiStore = DAG.getStore(
          Chain, DL, Hi, BasePtr,
          Store->getPointerInfo().getWithOffset(HalfSize),
          Store->getBaseAlign(), Store->getMemOperand()->getFlags());
 
      return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoStore, HiStore);
    }
 
    if (VT == MVT::bf16)
      return lowerXAndesBfHCvtBFloat16Store(Op, DAG);
 
    // Handle normal vector tuple store.
    if (VT.isRISCVVectorTuple()) {
      SDLoc DL(Op);
      MVT XLenVT = Subtarget.getXLenVT();
      unsigned NF = VT.getRISCVVectorTupleNumFields();
      unsigned Sz = VT.getSizeInBits().getKnownMinValue();
      unsigned NumElts = Sz / (NF * 8);
      int Log2LMUL = Log2_64(NumElts) - 3;
 
      auto Flag = SDNodeFlags();
      Flag.setNoUnsignedWrap(true);
      SDValue Ret;
      SDValue Chain = Store->getChain();
      SDValue BasePtr = Store->getBasePtr();
      SDValue VROffset = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
      VROffset =
          DAG.getNode(ISD::SHL, DL, XLenVT, VROffset,
                      DAG.getConstant(std::max(Log2LMUL, 0), DL, XLenVT));
 
      // Extract subregisters in a vector tuple and store them individually.
      for (unsigned i = 0; i < NF; ++i) {
        auto Extract =
            DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL,
                        MVT::getScalableVectorVT(MVT::i8, NumElts), StoredVal,
                        DAG.getTargetConstant(i, DL, MVT::i32));
        Ret = DAG.getStore(Chain, DL, Extract, BasePtr,
                           MachinePointerInfo(Store->getAddressSpace()),
                           Store->getBaseAlign(),
                           Store->getMemOperand()->getFlags());
        Chain = Ret.getValue(0);
        BasePtr = DAG.getNode(ISD::ADD, DL, XLenVT, BasePtr, VROffset, Flag);
      }
      return Ret;
    }
 
    if (auto V = expandUnalignedRVVStore(Op, DAG))
      return V;
    if (Op.getOperand(1).getValueType().isFixedLengthVector())
      return lowerFixedLengthVectorStoreToRVV(Op, DAG);
    return Op;
  }
  case ISD::VP_LOAD:
    if (SDValue V = expandUnalignedVPLoad(Op, DAG))
      return V;
    [[fallthrough]];
  case ISD::MLOAD:
    return lowerMaskedLoad(Op, DAG);
  case ISD::VP_LOAD_FF:
    return lowerLoadFF(Op, DAG);
  case ISD::VP_STORE:
    if (SDValue V = expandUnalignedVPStore(Op, DAG))
      return V;
    [[fallthrough]];
  case ISD::MSTORE:
    return lowerMaskedStore(Op, DAG);
  case ISD::VECTOR_COMPRESS:
    return lowerVectorCompress(Op, DAG);
  case ISD::SELECT_CC: {
    // This occurs because we custom legalize SETGT and SETUGT for setcc. That
    // causes LegalizeDAG to think we need to custom legalize select_cc. Expand
    // into separate SETCC+SELECT just like LegalizeDAG.
    SDValue Tmp1 = Op.getOperand(0);
    SDValue Tmp2 = Op.getOperand(1);
    SDValue True = Op.getOperand(2);
    SDValue False = Op.getOperand(3);
    EVT VT = Op.getValueType();
    SDValue CC = Op.getOperand(4);
    EVT CmpVT = Tmp1.getValueType();
    EVT CCVT =
        getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
    SDLoc DL(Op);
    SDValue Cond =
        DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags());
    return DAG.getSelect(DL, VT, Cond, True, False);
  }
  case ISD::SETCC: {
    MVT OpVT = Op.getOperand(0).getSimpleValueType();
    if (OpVT.isScalarInteger()) {
      MVT VT = Op.getSimpleValueType();
      SDValue LHS = Op.getOperand(0);
      SDValue RHS = Op.getOperand(1);
      ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get();
      assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) &&
             "Unexpected CondCode");
 
      SDLoc DL(Op);
 
      // If the RHS is a constant in the range [-2049, 0) or (0, 2046], we can
      // convert this to the equivalent of (set(u)ge X, C+1) by using
      // (xori (slti(u) X, C+1), 1). This avoids materializing a small constant
      // in a register.
      if (isa<ConstantSDNode>(RHS)) {
        int64_t Imm = cast<ConstantSDNode>(RHS)->getSExtValue();
        if (Imm != 0 && isInt<12>((uint64_t)Imm + 1)) {
          // If this is an unsigned compare and the constant is -1, incrementing
          // the constant would change behavior. The result should be false.
          if (CCVal == ISD::SETUGT && Imm == -1)
            return DAG.getConstant(0, DL, VT);
          // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
          CCVal = ISD::getSetCCSwappedOperands(CCVal);
          SDValue SetCC = DAG.getSetCC(
              DL, VT, LHS, DAG.getSignedConstant(Imm + 1, DL, OpVT), CCVal);
          return DAG.getLogicalNOT(DL, SetCC, VT);
        }
        // Lower (setugt X, 2047) as (setne (srl X, 11), 0).
        if (CCVal == ISD::SETUGT && Imm == 2047) {
          SDValue Shift = DAG.getNode(ISD::SRL, DL, OpVT, LHS,
                                      DAG.getShiftAmountConstant(11, OpVT, DL));
          return DAG.getSetCC(DL, VT, Shift, DAG.getConstant(0, DL, OpVT),
                              ISD::SETNE);
        }
      }
 
      // Not a constant we could handle, swap the operands and condition code to
      // SETLT/SETULT.
      CCVal = ISD::getSetCCSwappedOperands(CCVal);
      return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
    }
 
    MVT VT = Op.getSimpleValueType();
    if (Subtarget.hasStdExtP() && VT.isFixedLengthVector()) {
      ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get();
      SDValue LHS = Op.getOperand(0);
      SDValue RHS = Op.getOperand(1);
      SDLoc DL(Op);
      if (CCVal == ISD::SETNE) {
        // Convert setne X, 0 to setult 0, X.
        if (ISD::isConstantSplatVectorAllZeros(RHS.getNode())) {
          return DAG.getSetCC(DL, VT, RHS, LHS, ISD::SETULT);
        }
 
        // Not a constant we could handle, convert to SETEQ+Invert
        SDValue SetCC = DAG.getSetCC(DL, VT, LHS, RHS, ISD::SETEQ);
        return DAG.getLogicalNOT(DL, SetCC, VT);
      }
 
      if (CCVal == ISD::SETGT) {
        if (ISD::isConstantSplatVectorAllOnes(RHS.getNode())) {
          SDValue SetCC =
              DAG.getSetCC(DL, VT, LHS, DAG.getConstant(0, DL, VT), ISD::SETLT);
          return DAG.getLogicalNOT(DL, SetCC, VT);
        }
 
        // Not a constant we could handle, swap the operands and condition code
        // to SETLT.
        CCVal = ISD::getSetCCSwappedOperands(CCVal);
        return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
      }
 
      return SDValue();
    }
 
    if (isPromotedOpNeedingSplit(Op.getOperand(0), Subtarget, *this))
      return SplitVectorOp(Op, DAG);
 
    return lowerToScalableOp(Op, DAG);
  }
  case ISD::ADD:
  case ISD::SUB:
  case ISD::SDIV:
  case ISD::SREM:
  case ISD::UDIV:
  case ISD::UREM:
  case ISD::BSWAP:
  case ISD::CTPOP:
    return lowerToScalableOp(Op, DAG);
  case ISD::VSELECT: {
    EVT VT = Op.getValueType();
    // Split 64-bit vector VSELECT on RV32 with P extension for v4i16 and v8i8
    if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
        (VT == MVT::v4i16 || VT == MVT::v8i8)) {
      SDLoc DL(Op);
      SDValue Mask = Op.getOperand(0);
      SDValue TrueVal = Op.getOperand(1);
      SDValue FalseVal = Op.getOperand(2);
 
      // Split all three operands into two halves
      auto [MaskLo, MaskHi] = DAG.SplitVector(Mask, DL);
      auto [TrueLo, TrueHi] = DAG.SplitVector(TrueVal, DL);
      auto [FalseLo, FalseHi] = DAG.SplitVector(FalseVal, DL);
 
      // Perform VSELECT on each half
      SDValue ResLo = DAG.getNode(ISD::VSELECT, DL, TrueLo.getValueType(),
                                  MaskLo, TrueLo, FalseLo);
      SDValue ResHi = DAG.getNode(ISD::VSELECT, DL, TrueHi.getValueType(),
                                  MaskHi, TrueHi, FalseHi);
 
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
    }
    return lowerToScalableOp(Op, DAG);
  }
  case ISD::AND:
  case ISD::OR:
  case ISD::XOR:
  case ISD::MUL:
  case ISD::MULHS:
  case ISD::MULHU: {
    EVT VT = Op.getValueType();
    unsigned Opc = Op.getOpcode();
    // Split 64-bit vector AND/OR/XOR/MUL/MULHS/MULHU on RV32 with P extension
    if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
        (VT == MVT::v4i16 || VT == MVT::v8i8)) {
      SDLoc DL(Op);
      SDValue LHS = Op.getOperand(0);
      SDValue RHS = Op.getOperand(1);
 
      // Determine the half-size type
      MVT HalfVT = (VT == MVT::v4i16) ? MVT::v2i16 : MVT::v4i8;
 
      // Extract the two halves from LHS
      auto [LHSLo, LHSHi] = DAG.SplitVector(LHS, DL, HalfVT, HalfVT);
 
      // Extract the two halves from RHS
      auto [RHSLo, RHSHi] = DAG.SplitVector(RHS, DL, HalfVT, HalfVT);
 
      // Perform the operation on each half
      SDValue ResLo = DAG.getNode(Opc, DL, HalfVT, LHSLo, RHSLo);
      SDValue ResHi = DAG.getNode(Opc, DL, HalfVT, LHSHi, RHSHi);
 
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
    }
    // Lower v4i8/v2i16 MUL/MULHS/MULHU via widening multiply + srl + truncate.
    if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
        (Opc == ISD::MUL || Opc == ISD::MULHS || Opc == ISD::MULHU) &&
        (VT == MVT::v4i8 || VT == MVT::v2i16)) {
      assert((VT == MVT::v4i8 || Opc == ISD::MUL) &&
             "Unexpected custom legalisation");
      SDLoc DL(Op);
      MVT WideVT = (VT == MVT::v4i8) ? MVT::v4i16 : MVT::v2i32;
      unsigned WMulOpc =
          (Opc == ISD::MULHU) ? RISCVISD::PWMULU : RISCVISD::PWMUL;
      SDValue Res =
          DAG.getNode(WMulOpc, DL, WideVT, Op.getOperand(0), Op.getOperand(1));
      if (Opc != ISD::MUL) {
        unsigned EltBits = VT.getVectorElementType().getSizeInBits();
        Res = DAG.getNode(ISD::SRL, DL, WideVT, Res,
                          DAG.getConstant(EltBits, DL, WideVT));
      }
      return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
    }
    // Lower v8i8 MULHS/MULHU on RV64 via a pair of widening byte multiplies
    // (picking out the even/odd result lanes) recombined with PPAIRO.
    if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::v8i8 &&
        Opc != ISD::MUL) {
      SDLoc DL(Op);
      SDValue LHS = Op.getOperand(0);
      SDValue RHS = Op.getOperand(1);
      unsigned LoOpc = Opc == ISD::MULHU ? RISCVISD::PMULU_HALVES_00
                                         : RISCVISD::PMUL_HALVES_00;
      unsigned HiOpc = Opc == ISD::MULHU ? RISCVISD::PMULU_HALVES_11
                                         : RISCVISD::PMUL_HALVES_11;
      SDValue Lo = DAG.getNode(LoOpc, DL, MVT::v4i16, LHS, RHS);
      SDValue Hi = DAG.getNode(HiOpc, DL, MVT::v4i16, LHS, RHS);
      return DAG.getNode(RISCVISD::PPAIRO, DL, VT, DAG.getBitcast(VT, Lo),
                         DAG.getBitcast(VT, Hi));
    }
    return lowerToScalableOp(Op, DAG);
  }
  case ISD::ANY_EXTEND_VECTOR_INREG: {
    EVT VT = Op.getValueType();
    assert(Subtarget.hasStdExtP() && Subtarget.is64Bit() &&
           (VT == MVT::v2i32 || VT == MVT::v4i16) &&
           "Unexpected custom legalisation");
    return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), VT,
                       Op.getOperand(0));
  }
  case ISD::SHL:
  case ISD::SRL:
  case ISD::SRA:
  case ISD::SSHLSAT:
    if (Op.getSimpleValueType().isFixedLengthVector()) {
      if (Subtarget.hasStdExtP()) {
        SDValue ShAmtVec = Op.getOperand(1);
        SDValue SplatVal;
        if (ShAmtVec.getOpcode() == ISD::SPLAT_VECTOR)
          SplatVal = ShAmtVec.getOperand(0);
        else if (ShAmtVec.getOpcode() == ISD::BUILD_VECTOR)
          SplatVal = cast<BuildVectorSDNode>(ShAmtVec)->getSplatValue();
 
        if (!SplatVal)
          return SDValue();
 
        unsigned Opc;
        switch (Op.getOpcode()) {
        default:
          llvm_unreachable("Unexpected opcode");
        case ISD::SHL:
          Opc = RISCVISD::PSLL;
          break;
        case ISD::SRL:
          Opc = RISCVISD::PSRL;
          break;
        case ISD::SRA:
          Opc = RISCVISD::PSRA;
          break;
        case ISD::SSHLSAT:
          Opc = RISCVISD::PSSHA;
          break;
        }
        return DAG.getNode(Opc, SDLoc(Op), Op.getValueType(), Op.getOperand(0),
                           SplatVal);
      }
      return lowerToScalableOp(Op, DAG);
    }
    assert(Op.getOpcode() != ISD::SSHLSAT);
    // This can be called for an i32 shift amount that needs to be promoted.
    assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
    return SDValue();
  case ISD::MASKED_UDIV:
  case ISD::MASKED_SDIV:
  case ISD::MASKED_UREM:
  case ISD::MASKED_SREM: {
    SDLoc DL(Op);
    MVT VT = Op.getSimpleValueType();
    MVT ContainerVT = getContainerForFixedLengthVector(VT);
    SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
    SDValue Res = DAG.getNode(
        getRISCVVLOp(Op), DL, ContainerVT,
        convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget),
        convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget),
        DAG.getUNDEF(ContainerVT),
        convertToScalableVector(getMaskTypeFor(ContainerVT), Op.getOperand(2),
                                DAG, Subtarget),
        VL);
    return convertFromScalableVector(VT, Res, DAG, Subtarget);
  }
  case ISD::FABS:
  case ISD::FNEG:
    if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
      return lowerFABSorFNEG(Op, DAG, Subtarget);
    [[fallthrough]];
  case ISD::FADD:
  case ISD::FSUB:
  case ISD::FMUL:
  case ISD::FDIV:
  case ISD::FSQRT:
  case ISD::FMA:
  case ISD::FMINNUM:
  case ISD::FMAXNUM:
  case ISD::FMINIMUMNUM:
  case ISD::FMAXIMUMNUM:
    if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
      return SplitVectorOp(Op, DAG);
    [[fallthrough]];
  case ISD::AVGFLOORS:
  case ISD::AVGFLOORU:
  case ISD::AVGCEILS:
  case ISD::AVGCEILU:
  case ISD::SMIN:
  case ISD::SMAX:
  case ISD::UMIN:
  case ISD::UMAX:
  case ISD::UADDSAT:
  case ISD::USUBSAT:
  case ISD::SADDSAT:
  case ISD::SSUBSAT:
    return lowerToScalableOp(Op, DAG);
  case ISD::ABDS:
  case ISD::ABDU: {
    EVT VT = Op->getValueType(0);
    if (Subtarget.hasStdExtZvabd() && VT.isVector())
      return lowerToScalableOp(Op, DAG);
 
    SDLoc dl(Op);
    SDValue LHS = DAG.getFreeze(Op->getOperand(0));
    SDValue RHS = DAG.getFreeze(Op->getOperand(1));
    bool IsSigned = Op->getOpcode() == ISD::ABDS;
 
    // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
    // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
    unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
    unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
    SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
    SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
    return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
  }
  case ISD::ABS:
  case ISD::ABS_MIN_POISON:
    return lowerABS(Op, DAG);
  case ISD::CTLZ:
  case ISD::CTLZ_ZERO_POISON:
  case ISD::CTTZ:
  case ISD::CTTZ_ZERO_POISON:
    if (Subtarget.hasStdExtZvbb())
      return lowerToScalableOp(Op, DAG);
    assert(Op.getOpcode() != ISD::CTTZ);
    return lowerCTLZ_CTTZ_ZERO_POISON(Op, DAG);
  case ISD::CLMUL:
  case ISD::CLMULH: {
    SDLoc DL(Op);
    MVT VT = Op.getSimpleValueType();
    MVT XLenVT = Subtarget.getXLenVT();
 
    if (!VT.isVector()) {
      // The op needs to be performed in a vector register.
      assert(Subtarget.hasStdExtZvbc() && Subtarget.is64Bit() && VT == XLenVT &&
             "Unexpected custom legalisation");
      // We can't implicitly use vector registers with noimplicitfloat.
      if (DAG.getMachineFunction().getFunction().hasFnAttribute(
              Attribute::NoImplicitFloat))
        return SDValue();
      MVT VecVT = MVT::getScalableVectorVT(VT, 1);
      SDValue VL = DAG.getConstant(1, DL, XLenVT);
      SDValue Op0 =
          lowerScalarInsert(Op.getOperand(0), VL, VecVT, DL, DAG, Subtarget);
      SDValue Op1 =
          lowerScalarInsert(Op.getOperand(1), VL, VecVT, DL, DAG, Subtarget);
      SDValue Res = DAG.getNode(Op.getOpcode(), DL, VecVT, Op0, Op1);
      return DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Res);
    }
 
    // If the scalable vector version of this op is Legal, convert to scalable.
    if (VT.isFixedLengthVector() &&
        (VT.getVectorElementType() == MVT::i64 || Subtarget.hasStdExtZvbc32e()))
      return lowerToScalableOp(Op, DAG);
 
    assert(Op.getOpcode() == ISD::CLMUL && Subtarget.hasStdExtZvbc() &&
           "Unexpected custom legalisation");
    // Promote to i64 vector.
    MVT I64VecVT = VT.changeVectorElementType(MVT::i64);
    SDValue Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, I64VecVT, Op.getOperand(0));
    SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, I64VecVT, Op.getOperand(1));
    SDValue CLMUL = DAG.getNode(ISD::CLMUL, DL, I64VecVT, Op0, Op1);
    return DAG.getNode(ISD::TRUNCATE, DL, VT, CLMUL);
  }
  case ISD::FCOPYSIGN:
    if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
      return lowerFCOPYSIGN(Op, DAG, Subtarget);
    if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
      return SplitVectorOp(Op, DAG);
    return lowerToScalableOp(Op, DAG);
  case ISD::STRICT_FADD:
  case ISD::STRICT_FSUB:
  case ISD::STRICT_FMUL:
  case ISD::STRICT_FDIV:
  case ISD::STRICT_FSQRT:
  case ISD::STRICT_FMA:
    if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
      return SplitStrictFPVectorOp(Op, DAG);
    return lowerToScalableOp(Op, DAG);
  case ISD::STRICT_FSETCC:
  case ISD::STRICT_FSETCCS:
    return lowerVectorStrictFSetcc(Op, DAG);
  case ISD::STRICT_FCEIL:
  case ISD::STRICT_FRINT:
  case ISD::STRICT_FFLOOR:
  case ISD::STRICT_FTRUNC:
  case ISD::STRICT_FNEARBYINT:
  case ISD::STRICT_FROUND:
  case ISD::STRICT_FROUNDEVEN:
    return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
  case ISD::MGATHER:
  case ISD::VP_GATHER:
    return lowerMaskedGather(Op, DAG);
  case ISD::MSCATTER:
  case ISD::VP_SCATTER:
    return lowerMaskedScatter(Op, DAG);
  case ISD::GET_ROUNDING:
    return lowerGET_ROUNDING(Op, DAG);
  case ISD::SET_ROUNDING:
    return lowerSET_ROUNDING(Op, DAG);
  case ISD::GET_FPENV:
    return lowerGET_FPENV(Op, DAG);
  case ISD::SET_FPENV:
    return lowerSET_FPENV(Op, DAG);
  case ISD::RESET_FPENV:
    return lowerRESET_FPENV(Op, DAG);
  case ISD::GET_FPMODE:
    return lowerGET_FPMODE(Op, DAG);
  case ISD::SET_FPMODE:
    return lowerSET_FPMODE(Op, DAG);
  case ISD::RESET_FPMODE:
    return lowerRESET_FPMODE(Op, DAG);
  case ISD::EH_DWARF_CFA:
    return lowerEH_DWARF_CFA(Op, DAG);
  case ISD::VP_MERGE:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return lowerVPMergeMask(Op, DAG);
    [[fallthrough]];
  case ISD::VP_SDIV:
  case ISD::VP_UDIV:
  case ISD::VP_SREM:
  case ISD::VP_UREM:
    return lowerVPOp(Op, DAG);
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return lowerVPStridedLoad(Op, DAG);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return lowerVPStridedStore(Op, DAG);
  case ISD::EXPERIMENTAL_VP_SPLICE:
    return lowerVPSpliceExperimental(Op, DAG);
  case ISD::EXPERIMENTAL_VP_REVERSE:
    return lowerVPReverseExperimental(Op, DAG);
  case ISD::CLEAR_CACHE: {
    assert(getTargetMachine().getTargetTriple().isOSLinux() &&
           "llvm.clear_cache only needs custom lower on Linux targets");
    SDLoc DL(Op);
    SDValue Flags = DAG.getConstant(0, DL, Subtarget.getXLenVT());
    return emitFlushICache(DAG, Op.getOperand(0), Op.getOperand(1),
                           Op.getOperand(2), Flags, DL);
  }
  case ISD::DYNAMIC_STACKALLOC:
    return lowerDYNAMIC_STACKALLOC(Op, DAG);
  case ISD::INIT_TRAMPOLINE:
    return lowerINIT_TRAMPOLINE(Op, DAG);
  case ISD::ADJUST_TRAMPOLINE:
    return lowerADJUST_TRAMPOLINE(Op, DAG);
  case ISD::PARTIAL_REDUCE_UMLA:
  case ISD::PARTIAL_REDUCE_SMLA:
  case ISD::PARTIAL_REDUCE_SUMLA:
    return lowerPARTIAL_REDUCE_MLA(Op, DAG);
  case ISD::CTTZ_ELTS:
  case ISD::CTTZ_ELTS_ZERO_POISON:
    return lowerCttzElts(Op, DAG, Subtarget);
  case ISD::CONVERT_FROM_ARBITRARY_FP:
    return lowerCONVERT_FROM_ARBITRARY_FP(Op, DAG, Subtarget);
  }
}
 
SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
                                             SDValue Start, SDValue End,
                                             SDValue Flags, SDLoc DL) const {
  MakeLibCallOptions CallOptions;
  std::pair<SDValue, SDValue> CallResult =
      makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
                  {Start, End, Flags}, CallOptions, DL, InChain);
 
  // This function returns void so only the out chain matters.
  return CallResult.second;
}
 
SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
                                                  SelectionDAG &DAG) const {
  if (!Subtarget.is64Bit())
    llvm::reportFatalUsageError("Trampolines only implemented for RV64");
 
  // Create an MCCodeEmitter to encode instructions.
  TargetLoweringObjectFile *TLO = getTargetMachine().getObjFileLowering();
  assert(TLO);
  MCContext &MCCtx = TLO->getContext();
 
  std::unique_ptr<MCCodeEmitter> CodeEmitter(
      createRISCVMCCodeEmitter(*getTargetMachine().getMCInstrInfo(), MCCtx));
 
  SDValue Root = Op.getOperand(0);
  SDValue Trmp = Op.getOperand(1); // trampoline
  SDLoc dl(Op);
 
  const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
 
  // We store in the trampoline buffer the following instructions and data.
  // Offset:
  //      0: auipc   t2, 0
  //      4: ld      t0, 24(t2)
  //      8: ld      t2, 16(t2)
  //     12: jalr    t0
  //     16: <StaticChainOffset>
  //     24: <FunctionAddressOffset>
  //     32:
  // Offset with branch control flow protection enabled:
  //      0: lpad    <imm20>
  //      4: auipc   t3, 0
  //      8: ld      t2, 28(t3)
  //     12: ld      t3, 20(t3)
  //     16: jalr    t2
  //     20: <StaticChainOffset>
  //     28: <FunctionAddressOffset>
  //     36:
 
  const MachineFunction &MF = DAG.getMachineFunction();
  const bool HasCFBranch =
      MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
  const unsigned StaticChainIdx = HasCFBranch ? 5 : 4;
  const unsigned StaticChainOffset = StaticChainIdx * 4;
  const unsigned FunctionAddressOffset = StaticChainOffset + 8;
 
  const MCSubtargetInfo &STI = getTargetMachine().getMCSubtargetInfo();
  auto GetEncoding = [&](const MCInst &MC) {
    SmallVector<char, 4> CB;
    SmallVector<MCFixup> Fixups;
    CodeEmitter->encodeInstruction(MC, CB, Fixups, STI);
    uint32_t Encoding = support::endian::read32le(CB.data());
    return Encoding;
  };
 
  SmallVector<SDValue> OutChains;
 
  SmallVector<uint32_t> Encodings;
  if (!HasCFBranch) {
    Encodings.append(
        {// auipc t2, 0
         // Loads the current PC into t2.
         GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X7).addImm(0)),
         // ld t0, 24(t2)
         // Loads the function address into t0. Note that we are using offsets
         // pc-relative to the first instruction of the trampoline.
         GetEncoding(MCInstBuilder(RISCV::LD)
                         .addReg(RISCV::X5)
                         .addReg(RISCV::X7)
                         .addImm(FunctionAddressOffset)),
         // ld t2, 16(t2)
         // Load the value of the static chain.
         GetEncoding(MCInstBuilder(RISCV::LD)
                         .addReg(RISCV::X7)
                         .addReg(RISCV::X7)
                         .addImm(StaticChainOffset)),
         // jalr t0
         // Jump to the function.
         GetEncoding(MCInstBuilder(RISCV::JALR)
                         .addReg(RISCV::X0)
                         .addReg(RISCV::X5)
                         .addImm(0))});
  } else {
    Encodings.append(
        {// auipc x0, <imm20> (lpad <imm20>)
         // Landing pad.
         GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X0).addImm(0)),
         // auipc t3, 0
         // Loads the current PC into t3.
         GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X28).addImm(0)),
         // ld t2, (FunctionAddressOffset - 4)(t3)
         // Loads the function address into t2. Note that we are using offsets
         // pc-relative to the SECOND instruction of the trampoline.
         GetEncoding(MCInstBuilder(RISCV::LD)
                         .addReg(RISCV::X7)
                         .addReg(RISCV::X28)
                         .addImm(FunctionAddressOffset - 4)),
         // ld t3, (StaticChainOffset - 4)(t3)
         // Load the value of the static chain.
         GetEncoding(MCInstBuilder(RISCV::LD)
                         .addReg(RISCV::X28)
                         .addReg(RISCV::X28)
                         .addImm(StaticChainOffset - 4)),
         // jalr t2
         // Software-guarded jump to the function.
         GetEncoding(MCInstBuilder(RISCV::JALR)
                         .addReg(RISCV::X0)
                         .addReg(RISCV::X7)
                         .addImm(0))});
  }
 
  // Store encoded instructions.
  for (auto [Idx, Encoding] : llvm::enumerate(Encodings)) {
    SDValue Addr = Idx > 0 ? DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
                                         DAG.getConstant(Idx * 4, dl, MVT::i64))
                           : Trmp;
    OutChains.push_back(DAG.getTruncStore(
        Root, dl, DAG.getConstant(Encoding, dl, MVT::i64), Addr,
        MachinePointerInfo(TrmpAddr, Idx * 4), MVT::i32));
  }
 
  // Now store the variable part of the trampoline.
  SDValue FunctionAddress = Op.getOperand(2);
  SDValue StaticChain = Op.getOperand(3);
 
  // Store the given static chain and function pointer in the trampoline buffer.
  struct OffsetValuePair {
    const unsigned Offset;
    const SDValue Value;
    SDValue Addr = SDValue(); // Used to cache the address.
  } OffsetValues[] = {
      {StaticChainOffset, StaticChain},
      {FunctionAddressOffset, FunctionAddress},
  };
  for (auto &OffsetValue : OffsetValues) {
    SDValue Addr =
        DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
                    DAG.getConstant(OffsetValue.Offset, dl, MVT::i64));
    OffsetValue.Addr = Addr;
    OutChains.push_back(
        DAG.getStore(Root, dl, OffsetValue.Value, Addr,
                     MachinePointerInfo(TrmpAddr, OffsetValue.Offset)));
  }
 
  assert(OutChains.size() == StaticChainIdx + 2 &&
         "Size of OutChains mismatch");
  SDValue StoreToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
 
  // The end of instructions of trampoline is the same as the static chain
  // address that we computed earlier.
  SDValue EndOfTrmp = OffsetValues[0].Addr;
 
  // Call clear cache on the trampoline instructions.
  SDValue Chain = DAG.getNode(ISD::CLEAR_CACHE, dl, MVT::Other, StoreToken,
                              Trmp, EndOfTrmp);
 
  return Chain;
}
 
SDValue RISCVTargetLowering::lowerADJUST_TRAMPOLINE(SDValue Op,
                                                    SelectionDAG &DAG) const {
  if (!Subtarget.is64Bit())
    llvm::reportFatalUsageError("Trampolines only implemented for RV64");
 
  return Op.getOperand(0);
}
 
SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
                                                     SelectionDAG &DAG) const {
  // Currently, only the vdot4a and vdot4au case (from zvdot4a8i) should be
  // legal.
  // TODO: There are many other sub-cases we could potentially lower, are
  // any of them worthwhile?  Ex: via vredsum, vwredsum, vwwmaccu, etc..
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue Accum = Op.getOperand(0);
  SDValue A = Op.getOperand(1);
  SDValue B = Op.getOperand(2);
  MVT ArgVT = A.getSimpleValueType();
  assert(ArgVT == B.getSimpleValueType() &&
         ArgVT.getVectorElementType() == MVT::i8);
  (void)ArgVT;
 
  // vdot4a* only produces an i32 result. For an i64 accumulator, perform the
  // dot product into a fresh i32 accumulator (each result is the sum of four
  // i8 products, which cannot overflow i32), reduce those i32 partial sums
  // down to the accumulator's element count while still in i32 (a sum of eight
  // i8 products still cannot overflow i32), and only then extend to i64 and add
  // to the accumulator.
  if (VT.getVectorElementType() == MVT::i64) {
    assert(Accum.getSimpleValueType() == VT);
    // vdot4a* reduces each group of four i8 lanes into one i32 lane, so the
    // intermediate i32 result has 1/4 the element count of the i8 inputs.
    MVT DotVT = MVT::getVectorVT(
        MVT::i32, ArgVT.getVectorElementCount().divideCoefficientBy(4));
    SDValue Dot = DAG.getNode(Op.getOpcode(), DL, DotVT,
                              {DAG.getConstant(0, DL, DotVT), A, B});
    // The reduced i32 sums are signed for SMLA/SUMLA and unsigned for UMLA.
    unsigned ExtOpc = Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA
                          ? ISD::ZERO_EXTEND
                          : ISD::SIGN_EXTEND;
 
    MVT NarrowVT = VT.changeVectorElementType(MVT::i32);
    if (VT.isScalableVector() &&
        RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(NarrowVT))
            .second) {
      // When the accumulator is a single vector (LMUL 1), the i32 subvectors
      // are a fractional LMUL, so extracting the high subvector would need a
      // vslidedown. Widen the i32 sums to i64 first instead; the i64
      // subvectors are then register-aligned and the reduction is a plain add.
      MVT WideVT = DotVT.changeVectorElementType(MVT::i64);
      SDValue Wide = DAG.getNode(ExtOpc, DL, WideVT, Dot);
      unsigned Stride = VT.getVectorMinNumElements();
      SDValue Sum = DAG.getExtractSubvector(DL, VT, Wide, 0);
      for (unsigned I = 1, E = WideVT.getVectorMinNumElements() / Stride;
           I != E; ++I)
        Sum = DAG.getNode(ISD::ADD, DL, VT, Sum,
                          DAG.getExtractSubvector(DL, VT, Wide, I * Stride));
      return DAG.getNode(ISD::ADD, DL, VT, Accum, Sum);
    }
 
    // Add the i32 partial sums down to the accumulator's element count by
    // extracting and summing the subvectors (still in i32 to avoid a wider
    // extend), then extend once to i64 and accumulate.
    unsigned Stride = NarrowVT.getVectorMinNumElements();
    SDValue Sum = DAG.getExtractSubvector(DL, NarrowVT, Dot, 0);
    for (unsigned I = 1, E = DotVT.getVectorMinNumElements() / Stride; I != E;
         ++I)
      Sum = DAG.getNode(ISD::ADD, DL, NarrowVT, Sum,
                        DAG.getExtractSubvector(DL, NarrowVT, Dot, I * Stride));
    return DAG.getNode(ISD::ADD, DL, VT, Accum,
                       DAG.getNode(ExtOpc, DL, VT, Sum));
  }
 
  assert(Accum.getSimpleValueType() == VT &&
         VT.getVectorElementType() == MVT::i32);
 
  // The zvdot4a8i pseudos are defined with sources and destination both
  // being i32.  This cast is needed for correctness to avoid incorrect
  // .vx matching of i8 splats.
  A = DAG.getBitcast(VT, A);
  B = DAG.getBitcast(VT, B);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    Accum = convertToScalableVector(ContainerVT, Accum, DAG, Subtarget);
    A = convertToScalableVector(ContainerVT, A, DAG, Subtarget);
    B = convertToScalableVector(ContainerVT, B, DAG, Subtarget);
  }
 
  unsigned Opc;
  switch (Op.getOpcode()) {
  case ISD::PARTIAL_REDUCE_SMLA:
    Opc = RISCVISD::VDOT4A_VL;
    break;
  case ISD::PARTIAL_REDUCE_UMLA:
    Opc = RISCVISD::VDOT4AU_VL;
    break;
  case ISD::PARTIAL_REDUCE_SUMLA:
    Opc = RISCVISD::VDOT4ASU_VL;
    break;
  default:
    llvm_unreachable("Unexpected opcode");
  }
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
  SDValue Res = DAG.getNode(Opc, DL, ContainerVT, {A, B, Accum, Mask, VL});
  if (VT.isFixedLengthVector())
    Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
  return Res;
}
 
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty,
                             SelectionDAG &DAG, unsigned Flags) {
  return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
}
 
static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
                             SelectionDAG &DAG, unsigned Flags) {
  return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
                                   Flags);
}
 
static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
                             SelectionDAG &DAG, unsigned Flags) {
  return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
                                   N->getOffset(), Flags);
}
 
static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
                             SelectionDAG &DAG, unsigned Flags) {
  return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
}
 
static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL,
                                     EVT Ty, SelectionDAG &DAG) {
  RISCVConstantPoolValue *CPV = RISCVConstantPoolValue::Create(N->getGlobal());
  SDValue CPAddr = DAG.getTargetConstantPool(CPV, Ty, Align(8));
  SDValue LC = DAG.getNode(RISCVISD::LLA, DL, Ty, CPAddr);
  return DAG.getLoad(
      Ty, DL, DAG.getEntryNode(), LC,
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
}
 
static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL,
                                      EVT Ty, SelectionDAG &DAG) {
  RISCVConstantPoolValue *CPV =
      RISCVConstantPoolValue::Create(*DAG.getContext(), N->getSymbol());
  SDValue CPAddr = DAG.getTargetConstantPool(CPV, Ty, Align(8));
  SDValue LC = DAG.getNode(RISCVISD::LLA, DL, Ty, CPAddr);
  return DAG.getLoad(
      Ty, DL, DAG.getEntryNode(), LC,
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
}
 
template <class NodeTy>
SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
                                     bool IsLocal, bool IsExternWeak) const {
  SDLoc DL(N);
  EVT Ty = getPointerTy(DAG.getDataLayout());
 
  // When HWASAN is used and tagging of global variables is enabled
  // they should be accessed via the GOT, since the tagged address of a global
  // is incompatible with existing code models. This also applies to non-pic
  // mode.
  if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) {
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
    if (IsLocal && !Subtarget.allowTaggedGlobals())
      // Use PC-relative addressing to access the symbol. This generates the
      // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
      // %pcrel_lo(auipc)).
      return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
 
    // Use PC-relative addressing to access the GOT for this symbol, then load
    // the address from the GOT. This generates the pattern (PseudoLGA sym),
    // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
    SDValue Load =
        SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
    MachineFunction &MF = DAG.getMachineFunction();
    MachineMemOperand *MemOp = MF.getMachineMemOperand(
        MachinePointerInfo::getGOT(MF),
        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
            MachineMemOperand::MOInvariant,
        LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
    DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
    return Load;
  }
 
  switch (getTargetMachine().getCodeModel()) {
  default:
    reportFatalUsageError("Unsupported code model for lowering");
  case CodeModel::Small: {
    // Generate a sequence for accessing addresses within the first 2 GiB of
    // address space.
    if (Subtarget.hasVendorXqcili()) {
      // Use QC.E.LI to generate the address, as this is easier to relax than
      // LUI/ADDI.
      SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
      return DAG.getNode(RISCVISD::QC_E_LI, DL, Ty, Addr);
    }
 
    // This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
    SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
    SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
    SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
    return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNHi, AddrLo);
  }
  case CodeModel::Medium: {
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
    if (IsExternWeak) {
      // An extern weak symbol may be undefined, i.e. have value 0, which may
      // not be within 2GiB of PC, so use GOT-indirect addressing to access the
      // symbol. This generates the pattern (PseudoLGA sym), which expands to
      // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
      SDValue Load =
          SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
      MachineFunction &MF = DAG.getMachineFunction();
      MachineMemOperand *MemOp = MF.getMachineMemOperand(
          MachinePointerInfo::getGOT(MF),
          MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
              MachineMemOperand::MOInvariant,
          LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
      DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
      return Load;
    }
 
    // Generate a sequence for accessing addresses within any 2GiB range within
    // the address space. This generates the pattern (PseudoLLA sym), which
    // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
    return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
  }
  case CodeModel::Large: {
    if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N))
      return getLargeGlobalAddress(G, DL, Ty, DAG);
 
    // Using pc-relative mode for other node type.
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
    return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
  }
  }
}
 
SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
                                                SelectionDAG &DAG) const {
  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
  assert(N->getOffset() == 0 && "unexpected offset in global node");
  const GlobalValue *GV = N->getGlobal();
  bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(GV);
  return getAddr(N, DAG, IsLocal, GV->hasExternalWeakLinkage());
}
 
SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
                                               SelectionDAG &DAG) const {
  BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
 
  return getAddr(N, DAG);
}
 
SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
                                               SelectionDAG &DAG) const {
  ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
 
  return getAddr(N, DAG);
}
 
SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
                                            SelectionDAG &DAG) const {
  JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
 
  return getAddr(N, DAG);
}
 
SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
                                              SelectionDAG &DAG,
                                              bool UseGOT) const {
  SDLoc DL(N);
  EVT Ty = getPointerTy(DAG.getDataLayout());
  const GlobalValue *GV = N->getGlobal();
  MVT XLenVT = Subtarget.getXLenVT();
 
  if (UseGOT) {
    // Use PC-relative addressing to access the GOT for this TLS symbol, then
    // load the address from the GOT and add the thread pointer. This generates
    // the pattern (PseudoLA_TLS_IE sym), which expands to
    // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
    SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
    SDValue Load =
        SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
    MachineFunction &MF = DAG.getMachineFunction();
    MachineMemOperand *MemOp = MF.getMachineMemOperand(
        MachinePointerInfo::getGOT(MF),
        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
            MachineMemOperand::MOInvariant,
        LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
    DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
 
    // Add the thread pointer.
    SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
    return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
  }
 
  // Generate a sequence for accessing the address relative to the thread
  // pointer, with the appropriate adjustment for the thread pointer offset.
  // This generates the pattern
  // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
  SDValue AddrHi =
      DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
  SDValue AddrAdd =
      DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
  SDValue AddrLo =
      DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
 
  SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
  SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
  SDValue MNAdd =
      DAG.getNode(RISCVISD::ADD_TPREL, DL, Ty, MNHi, TPReg, AddrAdd);
  return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNAdd, AddrLo);
}
 
SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
                                               SelectionDAG &DAG) const {
  SDLoc DL(N);
  EVT Ty = getPointerTy(DAG.getDataLayout());
  IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
  const GlobalValue *GV = N->getGlobal();
 
  // Use a PC-relative addressing mode to access the global dynamic GOT address.
  // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
  // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
  SDValue Load =
      SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
 
  // Prepare argument list to generate call.
  ArgListTy Args;
  Args.emplace_back(Load, CallTy);
 
  // Setup call to __tls_get_addr.
  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(DL)
      .setChain(DAG.getEntryNode())
      .setLibCallee(CallingConv::C, CallTy,
                    DAG.getExternalSymbol("__tls_get_addr", Ty),
                    std::move(Args));
 
  return LowerCallTo(CLI).first;
}
 
SDValue RISCVTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
                                            SelectionDAG &DAG) const {
  SDLoc DL(N);
  EVT Ty = getPointerTy(DAG.getDataLayout());
  const GlobalValue *GV = N->getGlobal();
 
  // Use a PC-relative addressing mode to access the global dynamic GOT address.
  // This generates the pattern (PseudoLA_TLSDESC sym), which expands to
  //
  // auipc tX, %tlsdesc_hi(symbol)         // R_RISCV_TLSDESC_HI20(symbol)
  // lw    tY, tX, %tlsdesc_load_lo(label) // R_RISCV_TLSDESC_LOAD_LO12(label)
  // addi  a0, tX, %tlsdesc_add_lo(label)  // R_RISCV_TLSDESC_ADD_LO12(label)
  // jalr  t0, tY                          // R_RISCV_TLSDESC_CALL(label)
  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
  return SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLSDESC, DL, Ty, Addr), 0);
}
 
SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
                                                   SelectionDAG &DAG) const {
  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
  assert(N->getOffset() == 0 && "unexpected offset in global node");
 
  if (DAG.getTarget().useEmulatedTLS())
    return LowerToTLSEmulatedModel(N, DAG);
 
  TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
 
  if (DAG.getMachineFunction().getFunction().getCallingConv() ==
      CallingConv::GHC)
    reportFatalUsageError("In GHC calling convention TLS is not supported");
 
  SDValue Addr;
  switch (Model) {
  case TLSModel::LocalExec:
    Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
    break;
  case TLSModel::InitialExec:
    Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
    break;
  case TLSModel::LocalDynamic:
  case TLSModel::GeneralDynamic:
    Addr = DAG.getTarget().useTLSDESC() ? getTLSDescAddr(N, DAG)
                                        : getDynamicTLSAddr(N, DAG);
    break;
  }
 
  return Addr;
}
 
// Return true if Val is equal to (setcc LHS, RHS, CC).
// Return false if Val is the inverse of (setcc LHS, RHS, CC).
// Otherwise, return std::nullopt.
static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
                                      ISD::CondCode CC, SDValue Val) {
  assert(Val->getOpcode() == ISD::SETCC);
  SDValue LHS2 = Val.getOperand(0);
  SDValue RHS2 = Val.getOperand(1);
  ISD::CondCode CC2 = cast<CondCodeSDNode>(Val.getOperand(2))->get();
 
  if (LHS == LHS2 && RHS == RHS2) {
    if (CC == CC2)
      return true;
    if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
      return false;
  } else if (LHS == RHS2 && RHS == LHS2) {
    CC2 = ISD::getSetCCSwappedOperands(CC2);
    if (CC == CC2)
      return true;
    if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
      return false;
  }
 
  return std::nullopt;
}
 
static bool isSimm12Constant(SDValue V) {
  return isa<ConstantSDNode>(V) && V->getAsAPIntVal().isSignedIntN(12);
}
 
static SDValue lowerSelectToBinOp(SDNode *N, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  SDValue CondV = N->getOperand(0);
  SDValue TrueV = N->getOperand(1);
  SDValue FalseV = N->getOperand(2);
  MVT VT = N->getSimpleValueType(0);
  SDLoc DL(N);
 
  if (!Subtarget.hasConditionalMoveFusion()) {
    // (select c, -1, y) -> -c | y
    if (isAllOnesConstant(TrueV)) {
      SDValue Neg = DAG.getNegative(CondV, DL, VT);
      return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
    }
    // (select c, y, -1) -> (c-1) | y
    if (isAllOnesConstant(FalseV)) {
      SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
                                DAG.getAllOnesConstant(DL, VT));
      return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
    }
 
    const bool HasZicond = VT.isScalarInteger() && Subtarget.hasStdExtZicond();
 
    // (select c, 0, y) -> (c-1) & y
    if (isNullConstant(TrueV) && (!HasZicond || isSimm12Constant(FalseV))) {
      SDValue Neg =
          DAG.getNode(ISD::ADD, DL, VT, CondV, DAG.getAllOnesConstant(DL, VT));
      return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
    }
    if (isNullConstant(FalseV)) {
      if (auto *TrueC = dyn_cast<ConstantSDNode>(TrueV)) {
        // (select c, y, 0) -> (c * (y - 1)) + c
        int64_t MulImm = TrueC->getSExtValue();
        if (MulImm != INT64_MIN && isInt<12>(MulImm - 1) &&
            Subtarget.hasVendorXqciac())
          return DAG.getNode(RISCVISD::QC_MULIADD, DL, VT, CondV, CondV,
                             DAG.getSignedTargetConstant(MulImm - 1, DL, VT));
 
        // (select c, (1 << ShAmount), 0) -> c << ShAmount
        uint64_t TrueVal = TrueC->getZExtValue();
        if (isPowerOf2_64(TrueVal))
          return DAG.getNode(
              ISD::SHL, DL, VT, CondV,
              DAG.getShiftAmountConstant(Log2_64(TrueVal), VT, DL));
 
        // (select c, (1 << ShAmount) + 1, 0) -> (c << ShAmount) + c
        uint64_t TrueM1 = TrueVal - 1;
        if (isPowerOf2_64(TrueM1)) {
          unsigned ShAmount = Log2_64(TrueM1);
          if (Subtarget.hasShlAdd(ShAmount))
            return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, CondV,
                               DAG.getTargetConstant(ShAmount, DL, VT), CondV);
        }
      }
      // (select c, y, 0) -> -c & y
      if (!HasZicond || isSimm12Constant(TrueV)) {
        SDValue Neg = DAG.getNegative(CondV, DL, VT);
        return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
      }
    }
  }
 
  // select c, ~x, x --> xor -c, x
  if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
    const APInt &TrueVal = TrueV->getAsAPIntVal();
    const APInt &FalseVal = FalseV->getAsAPIntVal();
    if (~TrueVal == FalseVal) {
      SDValue Neg = DAG.getNegative(CondV, DL, VT);
      return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
    }
  }
 
  // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
  // when both truev and falsev are also setcc.
  if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
      FalseV.getOpcode() == ISD::SETCC) {
    SDValue LHS = CondV.getOperand(0);
    SDValue RHS = CondV.getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
 
    // (select x, x, y) -> x | y
    // (select !x, x, y) -> x & y
    if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, TrueV)) {
      return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
                         DAG.getFreeze(FalseV));
    }
    // (select x, y, x) -> x & y
    // (select !x, y, x) -> x | y
    if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, FalseV)) {
      return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
                         DAG.getFreeze(TrueV), FalseV);
    }
  }
 
  return SDValue();
}
 
// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
// being `0` or `-1`. In such cases we can replace `select` with `and`.
// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
// than `c0`?
static SDValue
foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  if (Subtarget.hasShortForwardBranchIALU())
    return SDValue();
 
  unsigned SelOpNo = 0;
  SDValue Sel = BO->getOperand(0);
  if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
    SelOpNo = 1;
    Sel = BO->getOperand(1);
  }
 
  if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
    return SDValue();
 
  unsigned ConstSelOpNo = 1;
  unsigned OtherSelOpNo = 2;
  if (!isa<ConstantSDNode>(Sel->getOperand(ConstSelOpNo))) {
    ConstSelOpNo = 2;
    OtherSelOpNo = 1;
  }
  SDValue ConstSelOp = Sel->getOperand(ConstSelOpNo);
  ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
  if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
    return SDValue();
 
  SDValue ConstBinOp = BO->getOperand(SelOpNo ^ 1);
  ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
  if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
    return SDValue();
 
  SDLoc DL(Sel);
  EVT VT = BO->getValueType(0);
 
  SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
  if (SelOpNo == 1)
    std::swap(NewConstOps[0], NewConstOps[1]);
 
  SDValue NewConstOp =
      DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
  if (!NewConstOp)
    return SDValue();
 
  const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
  if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
    return SDValue();
 
  SDValue OtherSelOp = Sel->getOperand(OtherSelOpNo);
  SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
  if (SelOpNo == 1)
    std::swap(NewNonConstOps[0], NewNonConstOps[1]);
  SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
 
  SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
  SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
  return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF);
}
 
SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
  SDValue CondV = Op.getOperand(0);
  SDValue TrueV = Op.getOperand(1);
  SDValue FalseV = Op.getOperand(2);
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  // Handle P extension packed types by bitcasting to an integer of
  // matching width and reusing the scalar selection mechanism.
  // Reachable cases:
  //   RV32: v4i8/v2i16          -> select on i32
  //   RV32: v8i8/v4i16          -> select on i64 (legalizes to two i32 selects)
  //   RV64: v8i8/v4i16/v2i32    -> select on i64
  if (Subtarget.isPExtPackedType(VT)) {
    MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
    SDValue TrueVInt = DAG.getBitcast(IntVT, TrueV);
    SDValue FalseVInt = DAG.getBitcast(IntVT, FalseV);
    SDValue ResultInt =
        DAG.getNode(ISD::SELECT, DL, IntVT, CondV, TrueVInt, FalseVInt);
    return DAG.getBitcast(VT, ResultInt);
  }
 
  // Lower vector SELECTs to VSELECTs by splatting the condition.
  if (VT.isVector()) {
    MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
    SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV);
    return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
  }
 
  // Try some other optimizations before falling back to generic lowering.
  if (SDValue V = lowerSelectToBinOp(Op.getNode(), DAG, Subtarget))
    return V;
 
  // When there is no cost for GPR <-> FPR, we can use zicond select for
  // floating value when CondV is int type
  bool FPinGPR = Subtarget.hasStdExtZfinx();
 
  // We can handle FGPR without spliting into hi/lo parts
  bool FitsInGPR = TypeSize::isKnownLE(VT.getSizeInBits(),
                                       Subtarget.getXLenVT().getSizeInBits());
 
  bool UseZicondForFPSel = Subtarget.hasStdExtZicond() && FPinGPR &&
                           VT.isFloatingPoint() && FitsInGPR;
 
  if (UseZicondForFPSel) {
 
    auto CastToInt = [&](SDValue V) -> SDValue {
      // Treat +0.0 as int 0 to enable single 'czero' instruction generation.
      if (isNullFPConstant(V))
        return DAG.getConstant(0, DL, XLenVT);
 
      if (VT == MVT::f16)
        return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, V);
 
      if (VT == MVT::f32 && Subtarget.is64Bit())
        return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, V);
 
      return DAG.getBitcast(XLenVT, V);
    };
 
    SDValue TrueVInt = CastToInt(TrueV);
    SDValue FalseVInt = CastToInt(FalseV);
 
    // Emit integer SELECT (lowers to Zicond)
    SDValue ResultInt =
        DAG.getNode(ISD::SELECT, DL, XLenVT, CondV, TrueVInt, FalseVInt);
 
    // Convert back to floating VT
    if (VT == MVT::f32 && Subtarget.is64Bit())
      return DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, VT, ResultInt);
 
    if (VT == MVT::f16)
      return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, ResultInt);
 
    return DAG.getBitcast(VT, ResultInt);
  }
 
  // When Zicond is present, emit CZERO_EQZ and CZERO_NEZ
  // nodes to implement the SELECT. Performing the lowering here allows for
  // greater control over when CZERO_{EQZ/NEZ} are used vs another branchless
  // sequence or RISCVISD::SELECT_CC node (branch-based select).
  if (Subtarget.hasStdExtZicond() && VT.isScalarInteger()) {
 
    // (select c, t, 0) -> (czero_eqz t, c)
    if (isNullConstant(FalseV))
      return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV);
    // (select c, 0, f) -> (czero_nez f, c)
    if (isNullConstant(TrueV))
      return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV);
 
    // Check to see if a given operation is a 'NOT', if so return the negated
    // operand
    auto getNotOperand = [](const SDValue &Op) -> std::optional<const SDValue> {
      using namespace llvm::SDPatternMatch;
      SDValue Xor;
      if (sd_match(Op, m_OneUse(m_Not(m_Value(Xor))))) {
        return Xor;
      }
      return std::nullopt;
    };
    // (select c, (and f, x), f) -> (or (and f, x), (czero_nez f, c))
    // (select c, (and f, ~x), f) -> (andn f, (czero_eqz x, c))
    if (TrueV.getOpcode() == ISD::AND &&
        (TrueV.getOperand(0) == FalseV || TrueV.getOperand(1) == FalseV)) {
      auto NotOperand = (TrueV.getOperand(0) == FalseV)
                            ? getNotOperand(TrueV.getOperand(1))
                            : getNotOperand(TrueV.getOperand(0));
      if (NotOperand) {
        SDValue CMOV =
            DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, *NotOperand, CondV);
        SDValue NOT = DAG.getNOT(DL, CMOV, VT);
        return DAG.getNode(ISD::AND, DL, VT, FalseV, NOT);
      }
      return DAG.getNode(
          ISD::OR, DL, VT, TrueV,
          DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
    }
 
    // (select c, t, (and t, x)) -> (or (czero_eqz t, c), (and t, x))
    // (select c, t, (and t, ~x)) -> (andn t, (czero_nez x, c))
    if (FalseV.getOpcode() == ISD::AND &&
        (FalseV.getOperand(0) == TrueV || FalseV.getOperand(1) == TrueV)) {
      auto NotOperand = (FalseV.getOperand(0) == TrueV)
                            ? getNotOperand(FalseV.getOperand(1))
                            : getNotOperand(FalseV.getOperand(0));
      if (NotOperand) {
        SDValue CMOV =
            DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, *NotOperand, CondV);
        SDValue NOT = DAG.getNOT(DL, CMOV, VT);
        return DAG.getNode(ISD::AND, DL, VT, TrueV, NOT);
      }
      return DAG.getNode(
          ISD::OR, DL, VT, FalseV,
          DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV));
    }
 
    // (select c, c1, c2) -> (add (czero_nez c2 - c1, c), c1)
    // (select c, c1, c2) -> (add (czero_eqz c1 - c2, c), c2)
    if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
      const APInt &TrueVal = TrueV->getAsAPIntVal();
      const APInt &FalseVal = FalseV->getAsAPIntVal();
 
      // Prefer these over Zicond to avoid materializing an immediate:
      //   (select (x < 0), y, z)  -> x >> (XLEN - 1) & (y - z) + z
      //   (select (x > -1), z, y) -> x >> (XLEN - 1) & (y - z) + z
      if (CondV.getOpcode() == ISD::SETCC &&
          CondV.getOperand(0).getValueType() == VT && CondV.hasOneUse()) {
        ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
        if ((CCVal == ISD::SETLT && isNullConstant(CondV.getOperand(1))) ||
            (CCVal == ISD::SETGT && isAllOnesConstant(CondV.getOperand(1)))) {
          int64_t TrueImm = TrueVal.getSExtValue();
          int64_t FalseImm = FalseVal.getSExtValue();
          if (CCVal == ISD::SETGT)
            std::swap(TrueImm, FalseImm);
          if (isInt<12>(TrueImm) && isInt<12>(FalseImm) &&
              isInt<12>(TrueImm - FalseImm)) {
            SDValue SRA =
                DAG.getNode(ISD::SRA, DL, VT, CondV.getOperand(0),
                            DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
            SDValue AND =
                DAG.getNode(ISD::AND, DL, VT, SRA,
                            DAG.getSignedConstant(TrueImm - FalseImm, DL, VT));
            return DAG.getNode(ISD::ADD, DL, VT, AND,
                               DAG.getSignedConstant(FalseImm, DL, VT));
          }
        }
      }
 
      // Use SLLI/ADDI (and possible XORI) to avoid having to materialize
      // a constant in register
      if ((TrueVal - FalseVal).isPowerOf2() && FalseVal.isSignedIntN(12)) {
        SDValue Log2 = DAG.getConstant((TrueVal - FalseVal).logBase2(), DL, VT);
        SDValue BitDiff = DAG.getNode(ISD::SHL, DL, VT, CondV, Log2);
        return DAG.getNode(ISD::ADD, DL, VT, BitDiff, FalseV);
      }
      if ((FalseVal - TrueVal).isPowerOf2() && TrueVal.isSignedIntN(12)) {
        SDValue Log2 = DAG.getConstant((FalseVal - TrueVal).logBase2(), DL, VT);
        CondV = DAG.getLogicalNOT(DL, CondV, CondV.getValueType());
        SDValue BitDiff = DAG.getNode(ISD::SHL, DL, VT, CondV, Log2);
        return DAG.getNode(ISD::ADD, DL, VT, BitDiff, TrueV);
      }
 
      // If we can't use ADDI, it might still be profitable to use SLLI/ADD or
      // SLLI/SUB. We need to materialize a large constant for the czero+add
      // sequence below anyway. Using SLLI avoids materializing the delta.
      // Don't do this if the condition is an equality comparison since czero
      // allows us to fold part of the compare.
      if ((TrueVal - FalseVal).isPowerOf2() &&
          !(CondV.getOpcode() == ISD::SETCC &&
            ISD::isIntEqualitySetCC(
                cast<CondCodeSDNode>(CondV.getOperand(2))->get()))) {
        SDValue Log2 = DAG.getConstant((TrueVal - FalseVal).logBase2(), DL, VT);
        SDValue BitDiff = DAG.getNode(ISD::SHL, DL, VT, CondV, Log2);
        return DAG.getNode(ISD::ADD, DL, VT, FalseV, BitDiff);
      }
      if ((FalseVal - TrueVal).isPowerOf2() && !FalseVal.isSignedIntN(12) &&
          !(CondV.getOpcode() == ISD::SETCC &&
            ISD::isIntEqualitySetCC(
                cast<CondCodeSDNode>(CondV.getOperand(2))->get()))) {
        SDValue Log2 = DAG.getConstant((FalseVal - TrueVal).logBase2(), DL, VT);
        SDValue BitDiff = DAG.getNode(ISD::SHL, DL, VT, CondV, Log2);
        return DAG.getNode(ISD::SUB, DL, VT, FalseV, BitDiff);
      }
 
      auto getCost = [&](const APInt &Delta, const APInt &Addend) {
        const int DeltaCost = RISCVMatInt::getIntMatCost(
            Delta, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true);
        // Does the addend fold into an ADDI
        if (Addend.isSignedIntN(12))
          return DeltaCost;
        const int AddendCost = RISCVMatInt::getIntMatCost(
            Addend, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true);
        return AddendCost + DeltaCost;
      };
      bool IsCZERO_NEZ = getCost(FalseVal - TrueVal, TrueVal) <=
                         getCost(TrueVal - FalseVal, FalseVal);
      SDValue LHSVal = DAG.getConstant(
          IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
      SDValue CMOV =
          DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
                      DL, VT, LHSVal, CondV);
      return DAG.getNode(ISD::ADD, DL, VT, CMOV, IsCZERO_NEZ ? TrueV : FalseV);
    }
 
    // (select c, c1, t) -> (add (czero_nez t - c1, c), c1)
    // (select c, t, c1) -> (add (czero_eqz t - c1, c), c1)
    if (isa<ConstantSDNode>(TrueV) != isa<ConstantSDNode>(FalseV)) {
      bool IsCZERO_NEZ = isa<ConstantSDNode>(TrueV);
      SDValue ConstVal = IsCZERO_NEZ ? TrueV : FalseV;
      SDValue RegV = IsCZERO_NEZ ? FalseV : TrueV;
      int64_t RawConstVal = cast<ConstantSDNode>(ConstVal)->getSExtValue();
      // Efficient only if the constant and its negation fit into `ADDI`
      // Prefer Add/Sub over Xor since can be compressed for small immediates
      if (isInt<12>(RawConstVal)) {
        // Fall back to XORI if Const == -0x800 since we don't have SUBI.
        unsigned SubOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::SUB;
        unsigned AddOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::ADD;
        SDValue SubOp = DAG.getNode(SubOpc, DL, VT, RegV, ConstVal);
        SDValue CZERO =
            DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
                        DL, VT, SubOp, CondV);
        return DAG.getNode(AddOpc, DL, VT, CZERO, ConstVal);
      }
    }
 
    // Unless we have the short forward branch optimization, or we are
    // optimizing for size.
    if (!Subtarget.hasConditionalMoveFusion() && !DAG.shouldOptForSize()) {
      using namespace llvm::SDPatternMatch;
 
      // (select c, x, -x) -> (sub (czero_eqz x, c), (czero_nez x, c))
      if (sd_match(FalseV, m_Neg(m_Specific(TrueV)))) {
        TrueV = DAG.getFreeze(TrueV);
        return DAG.getNode(
            ISD::SUB, DL, VT,
            DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
            DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, TrueV, CondV));
      }
      // (select c, -x, x) -> (sub (czero_nez x, c), (czero_eqz x, c))
      if (sd_match(TrueV, m_Neg(m_Specific(FalseV)))) {
        FalseV = DAG.getFreeze(FalseV);
        return DAG.getNode(
            ISD::SUB, DL, VT,
            DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV),
            DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, FalseV, CondV));
      }
 
      // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
      return DAG.getNode(
          ISD::OR, DL, VT,
          DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
          DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV),
          SDNodeFlags::Disjoint);
    }
  }
 
  if (Op.hasOneUse()) {
    unsigned UseOpc = Op->user_begin()->getOpcode();
    if (isBinOp(UseOpc) && DAG.isSafeToSpeculativelyExecute(UseOpc)) {
      SDNode *BinOp = *Op->user_begin();
      if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(*Op->user_begin(),
                                                           DAG, Subtarget)) {
        DAG.ReplaceAllUsesWith(BinOp, &NewSel);
        // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
        // may return a constant node and cause crash in lowerSELECT.
        if (NewSel.getOpcode() == ISD::SELECT)
          return lowerSELECT(NewSel, DAG);
        return NewSel;
      }
    }
  }
 
  // (select cc, 1.0, 0.0) -> (sint_to_fp (zext cc))
  // (select cc, 0.0, 1.0) -> (sint_to_fp (zext (xor cc, 1)))
  const ConstantFPSDNode *FPTV = dyn_cast<ConstantFPSDNode>(TrueV);
  const ConstantFPSDNode *FPFV = dyn_cast<ConstantFPSDNode>(FalseV);
  if (FPTV && FPFV) {
    if (FPTV->isOne() && FPFV->isPosZero())
      return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV);
    if (FPTV->isPosZero() && FPFV->isOne()) {
      SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV,
                                DAG.getConstant(1, DL, XLenVT));
      return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR);
    }
  }
 
  // If the condition is not an integer SETCC which operates on XLenVT, we need
  // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
  // (select condv, truev, falsev)
  // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
  if (CondV.getOpcode() != ISD::SETCC ||
      CondV.getOperand(0).getSimpleValueType() != XLenVT) {
    SDValue Zero = DAG.getConstant(0, DL, XLenVT);
    SDValue SetNE = DAG.getCondCode(ISD::SETNE);
 
    SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
 
    return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
  }
 
  // If the CondV is the output of a SETCC node which operates on XLenVT inputs,
  // then merge the SETCC node into the lowered RISCVISD::SELECT_CC to take
  // advantage of the integer compare+branch instructions. i.e.:
  // (select (setcc lhs, rhs, cc), truev, falsev)
  // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
  SDValue LHS = CondV.getOperand(0);
  SDValue RHS = CondV.getOperand(1);
  ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
 
  // Special case for a select of 2 constants that have a difference of 1.
  // Normally this is done by DAGCombine, but if the select is introduced by
  // type legalization or op legalization, we miss it. Restricting to SETLT
  // case for now because that is what signed saturating add/sub need.
  // FIXME: We don't need the condition to be SETLT or even a SETCC,
  // but we would probably want to swap the true/false values if the condition
  // is SETGE/SETLE to avoid an XORI.
  if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
      CCVal == ISD::SETLT) {
    const APInt &TrueVal = TrueV->getAsAPIntVal();
    const APInt &FalseVal = FalseV->getAsAPIntVal();
    if (TrueVal - 1 == FalseVal)
      return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
    if (TrueVal + 1 == FalseVal)
      return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
  }
 
  translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
  // 1 < x ? x : 1 -> 0 < x ? x : 1
  if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
      RHS == TrueV && LHS == FalseV) {
    LHS = DAG.getConstant(0, DL, VT);
    // 0 <u x is the same as x != 0.
    if (CCVal == ISD::SETULT) {
      std::swap(LHS, RHS);
      CCVal = ISD::SETNE;
    }
  }
 
  // x <s -1 ? x : -1 -> x <s 0 ? x : -1
  if (isAllOnesConstant(RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
      RHS == FalseV) {
    RHS = DAG.getConstant(0, DL, VT);
  }
 
  SDValue TargetCC = DAG.getCondCode(CCVal);
 
  if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
    // (select (setcc lhs, rhs, CC), constant, falsev)
    // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
    std::swap(TrueV, FalseV);
    TargetCC = DAG.getCondCode(ISD::getSetCCInverse(CCVal, LHS.getValueType()));
  }
 
  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
  return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
}
 
SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
  SDValue CondV = Op.getOperand(1);
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
 
  if (CondV.getOpcode() == ISD::SETCC &&
      CondV.getOperand(0).getValueType() == XLenVT) {
    SDValue LHS = CondV.getOperand(0);
    SDValue RHS = CondV.getOperand(1);
    ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
 
    translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
 
    SDValue TargetCC = DAG.getCondCode(CCVal);
    return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
                       LHS, RHS, TargetCC, Op.getOperand(2));
  }
 
  return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
                     CondV, DAG.getConstant(0, DL, XLenVT),
                     DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
}
 
SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
 
  SDLoc DL(Op);
  SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
                                 getPointerTy(MF.getDataLayout()));
 
  // vastart just stores the address of the VarArgsFrameIndex slot into the
  // memory location argument.
  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
  return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
                      MachinePointerInfo(SV));
}
 
SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
                                            SelectionDAG &DAG) const {
  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  MFI.setFrameAddressIsTaken(true);
  Register FrameReg = RI.getFrameRegister(MF);
  int XLenInBytes = Subtarget.getXLen() / 8;
 
  EVT VT = Op.getValueType();
  SDLoc DL(Op);
  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
  unsigned Depth = Op.getConstantOperandVal(0);
  while (Depth--) {
    int Offset = -(XLenInBytes * 2);
    SDValue Ptr = DAG.getNode(
        ISD::ADD, DL, VT, FrameAddr,
        DAG.getSignedConstant(Offset, DL, getPointerTy(DAG.getDataLayout())));
    FrameAddr =
        DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
  }
  return FrameAddr;
}
 
SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
                                             SelectionDAG &DAG) const {
  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  MFI.setReturnAddressIsTaken(true);
  MVT XLenVT = Subtarget.getXLenVT();
  int XLenInBytes = Subtarget.getXLen() / 8;
 
  EVT VT = Op.getValueType();
  SDLoc DL(Op);
  unsigned Depth = Op.getConstantOperandVal(0);
  if (Depth) {
    int Off = -XLenInBytes;
    SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
    SDValue Offset = DAG.getSignedConstant(Off, DL, VT);
    return DAG.getLoad(VT, DL, DAG.getEntryNode(),
                       DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
                       MachinePointerInfo());
  }
 
  // Return the value of the return address register, marking it an implicit
  // live-in.
  Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
}
 
SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Lo = Op.getOperand(0);
  SDValue Hi = Op.getOperand(1);
  SDValue Shamt = Op.getOperand(2);
  EVT VT = Lo.getValueType();
  unsigned XLen = Subtarget.getXLen();
 
  // With P extension, use SLX (FSHL) for the high part.
  if (Subtarget.hasStdExtP()) {
    // HiRes = fshl(Hi, Lo, Shamt) - correct when Shamt < XLen
    SDValue HiRes = DAG.getNode(ISD::FSHL, DL, VT, Hi, Lo, Shamt);
    // LoRes = Lo << Shamt - correct Lo when Shamt < XLen,
    // Mask shift amount to avoid UB when Shamt >= XLen.
    SDValue ShamtMasked =
        DAG.getNode(ISD::AND, DL, VT, Shamt, DAG.getConstant(XLen - 1, DL, VT));
    SDValue LoRes = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMasked);
 
    // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
    // FIXME: We should use a select and let LowerSelect make the
    // optimizations.
    SDValue ShAmtExt =
        DAG.getNode(ISD::SHL, DL, VT, Shamt,
                    DAG.getConstant(XLen - Log2_32(XLen) - 1, DL, VT));
    SDValue Mask = DAG.getNode(ISD::SRA, DL, VT, ShAmtExt,
                               DAG.getConstant(XLen - 1, DL, VT));
 
    // When Shamt >= XLen: HiRes = LoRes, LoRes = 0
    // HiRes = (HiRes & ~Mask) | (LoRes & Mask)
    SDValue HiMasked =
        DAG.getNode(ISD::AND, DL, VT, HiRes, DAG.getNOT(DL, Mask, VT));
    SDValue LoMasked = DAG.getNode(ISD::AND, DL, VT, LoRes, Mask);
    HiRes =
        DAG.getNode(ISD::OR, DL, VT, HiMasked, LoMasked, SDNodeFlags::Disjoint);
 
    // LoRes = LoRes & ~Mask (clear when Shamt >= XLen)
    LoRes = DAG.getNode(ISD::AND, DL, VT, LoRes, DAG.getNOT(DL, Mask, VT));
 
    return DAG.getMergeValues({LoRes, HiRes}, DL);
  }
 
  // if Shamt-XLEN < 0: // Shamt < XLEN
  //   Lo = Lo << Shamt
  //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
  // else:
  //   Lo = 0
  //   Hi = Lo << (Shamt-XLEN)
 
  SDValue Zero = DAG.getConstant(0, DL, VT);
  SDValue One = DAG.getConstant(1, DL, VT);
  SDValue MinusXLen = DAG.getSignedConstant(-(int)XLen, DL, VT);
  SDValue XLenMinus1 = DAG.getConstant(XLen - 1, DL, VT);
  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
 
  SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
  SDValue ShiftRightLo =
      DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
  SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
  SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
 
  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
 
  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
 
  SDValue Parts[2] = {Lo, Hi};
  return DAG.getMergeValues(Parts, DL);
}
 
SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
                                                  bool IsSRA) const {
  SDLoc DL(Op);
  SDValue Lo = Op.getOperand(0);
  SDValue Hi = Op.getOperand(1);
  SDValue Shamt = Op.getOperand(2);
  EVT VT = Lo.getValueType();
 
  // With P extension, use NSRL/NSRA for RV32 or FSHR (SRX) for RV64.
  if (Subtarget.hasStdExtP()) {
    unsigned XLen = Subtarget.getXLen();
 
    SDValue LoRes;
    if (Subtarget.is64Bit()) {
      // On RV64, use FSHR (SRX instruction) for the low part. We will need
      // to fix this later if ShAmt >= 64.
      LoRes = DAG.getNode(ISD::FSHR, DL, VT, Hi, Lo, Shamt);
    } else {
      // On RV32, use NSRL/NSRA for the low part.
      // NSRL/NSRA read 6 bits of shift amount, so they handle Shamt >= 32
      // correctly.
      LoRes = DAG.getNode(IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL, VT, Lo,
                          Hi, Shamt);
    }
 
    // Mask shift amount to avoid UB when Shamt >= XLen.
    SDValue ShamtMasked =
        DAG.getNode(ISD::AND, DL, VT, Shamt, DAG.getConstant(XLen - 1, DL, VT));
    SDValue HiRes =
        DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, ShamtMasked);
 
    // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
    // FIXME: We should use a select and let LowerSelect make the
    // optimizations.
    SDValue ShAmtExt =
        DAG.getNode(ISD::SHL, DL, VT, Shamt,
                    DAG.getConstant(XLen - Log2_32(XLen) - 1, DL, VT));
    SDValue Mask = DAG.getNode(ISD::SRA, DL, VT, ShAmtExt,
                               DAG.getConstant(XLen - 1, DL, VT));
 
    if (Subtarget.is64Bit()) {
      // On RV64, FSHR masks shift amount to 63. We need to replace LoRes
      // with HiRes when Shamt >= 64.
      // LoRes = (LoRes & ~Mask) | (HiRes & Mask)
      SDValue LoMasked =
          DAG.getNode(ISD::AND, DL, VT, LoRes, DAG.getNOT(DL, Mask, VT));
      SDValue HiMasked = DAG.getNode(ISD::AND, DL, VT, HiRes, Mask);
      LoRes = DAG.getNode(ISD::OR, DL, VT, LoMasked, HiMasked,
                          SDNodeFlags::Disjoint);
    }
 
    // If ShAmt >= XLen, we need to replace HiRes with 0 or sign bits.
    if (IsSRA) {
      // sra hi, hi, (mask & (XLen-1)) - shifts by XLen-1 when shamt >= XLen
      SDValue MaskAmt = DAG.getNode(ISD::AND, DL, VT, Mask,
                                    DAG.getConstant(XLen - 1, DL, VT));
      HiRes = DAG.getNode(ISD::SRA, DL, VT, HiRes, MaskAmt);
    } else {
      // andn hi, hi, mask - clears hi when shamt >= XLen
      HiRes = DAG.getNode(ISD::AND, DL, VT, HiRes, DAG.getNOT(DL, Mask, VT));
    }
 
    return DAG.getMergeValues({LoRes, HiRes}, DL);
  }
 
  // SRA expansion:
  //   if Shamt-XLEN < 0: // Shamt < XLEN
  //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
  //     Hi = Hi >>s Shamt
  //   else:
  //     Lo = Hi >>s (Shamt-XLEN);
  //     Hi = Hi >>s (XLEN-1)
  //
  // SRL expansion:
  //   if Shamt-XLEN < 0: // Shamt < XLEN
  //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
  //     Hi = Hi >>u Shamt
  //   else:
  //     Lo = Hi >>u (Shamt-XLEN);
  //     Hi = 0;
 
  unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
 
  SDValue Zero = DAG.getConstant(0, DL, VT);
  SDValue One = DAG.getConstant(1, DL, VT);
  SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
 
  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
  SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
  SDValue ShiftLeftHi =
      DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
  SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
  SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
  SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
  SDValue HiFalse =
      IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
 
  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
 
  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
 
  SDValue Parts[2] = {Lo, Hi};
  return DAG.getMergeValues(Parts, DL);
}
 
// Lower splats of i1 types to SETCC. For each mask vector type, we have a
// legal equivalently-sized i8 type, so we can use that as a go-between.
SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
                                                  SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue SplatVal = Op.getOperand(0);
  // All-zeros or all-ones splats are handled specially.
  if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
    SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
    return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
  }
  if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
    SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
    return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
  }
  MVT InterVT = VT.changeVectorElementType(MVT::i8);
  SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal,
                         DAG.getConstant(1, DL, SplatVal.getValueType()));
  SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
  SDValue Zero = DAG.getConstant(0, DL, InterVT);
  return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
}
 
// Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
// illegal (currently only vXi64 RV32).
// FIXME: We could also catch non-constant sign-extended i32 values and lower
// them to VMV_V_X_VL.
SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
                                                     SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VecVT = Op.getSimpleValueType();
  assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
         "Unexpected SPLAT_VECTOR_PARTS lowering");
 
  assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
  SDValue Lo = Op.getOperand(0);
  SDValue Hi = Op.getOperand(1);
 
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VecVT);
 
  auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
 
  SDValue Res =
      splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG);
 
  if (VecVT.isFixedLengthVector())
    Res = convertFromScalableVector(VecVT, Res, DAG, Subtarget);
 
  return Res;
}
 
// Custom-lower extensions from mask vectors by using a vselect either with 1
// for zero/any-extension or -1 for sign-extension:
//   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
// Note that any-extension is lowered identically to zero-extension.
SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
                                                int64_t ExtTrueVal) const {
  SDLoc DL(Op);
  MVT VecVT = Op.getSimpleValueType();
  SDValue Src = Op.getOperand(0);
  // Only custom-lower extensions from mask types
  assert(Src.getValueType().isVector() &&
         Src.getValueType().getVectorElementType() == MVT::i1);
 
  if (VecVT.isScalableVector()) {
    SDValue SplatZero = DAG.getConstant(0, DL, VecVT);
    SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, VecVT);
    if (Src.getOpcode() == ISD::XOR &&
        ISD::isConstantSplatVectorAllOnes(Src.getOperand(1).getNode()))
      return DAG.getNode(ISD::VSELECT, DL, VecVT, Src.getOperand(0), SplatZero,
                         SplatTrueVal);
    return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
  }
 
  MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
  MVT I1ContainerVT =
      MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
 
  SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
 
  SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
 
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
  SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, XLenVT);
 
  if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
    SDValue Xor = Src.getOperand(0);
    if (Xor.getOpcode() == RISCVISD::VMXOR_VL) {
      SDValue ScalableOnes = Xor.getOperand(1);
      if (ScalableOnes.getOpcode() == ISD::INSERT_SUBVECTOR &&
          ScalableOnes.getOperand(0).isUndef() &&
          ISD::isConstantSplatVectorAllOnes(
              ScalableOnes.getOperand(1).getNode())) {
        CC = Xor.getOperand(0);
        std::swap(SplatZero, SplatTrueVal);
      }
    }
  }
 
  SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                          DAG.getUNDEF(ContainerVT), SplatZero, VL);
  SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                             DAG.getUNDEF(ContainerVT), SplatTrueVal, VL);
  SDValue Select =
      DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal,
                  SplatZero, DAG.getUNDEF(ContainerVT), VL);
 
  return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
}
 
// Custom-lower truncations from vectors to mask vectors by using a mask and a
// setcc operation:
//   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
                                                  SelectionDAG &DAG) const {
  SDLoc DL(Op);
  EVT MaskVT = Op.getValueType();
  // Only expect to custom-lower truncations to mask types
  assert(MaskVT.isVectorOf(MVT::i1) &&
         "Unexpected type for vector mask lowering");
  SDValue Src = Op.getOperand(0);
  MVT VecVT = Src.getSimpleValueType();
  // If this is a fixed vector, we need to convert it to a scalable vector.
  MVT ContainerVT = VecVT;
 
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
  SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
  SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
 
  SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                         DAG.getUNDEF(ContainerVT), SplatOne, VL);
  SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                          DAG.getUNDEF(ContainerVT), SplatZero, VL);
 
  MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
  SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
                              DAG.getUNDEF(ContainerVT), Mask, VL);
  Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT,
                      {Trunc, SplatZero, DAG.getCondCode(ISD::SETNE),
                       DAG.getUNDEF(MaskContainerVT), Mask, VL});
  if (MaskVT.isFixedLengthVector())
    Trunc = convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
  return Trunc;
}
 
SDValue RISCVTargetLowering::lowerVectorTrunc(SDValue Op,
                                              SelectionDAG &DAG) const {
  unsigned Opc = Op.getOpcode();
  SDLoc DL(Op);
 
  MVT VT = Op.getSimpleValueType();
  // Only custom-lower vector truncates
  assert(VT.isVector() && "Unexpected type for vector truncate lowering");
 
  // Truncates to mask types are handled differently
  if (VT.getVectorElementType() == MVT::i1)
    return lowerVectorMaskTrunc(Op, DAG);
 
  // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
  // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
  // truncate by one power of two at a time.
  MVT DstEltVT = VT.getVectorElementType();
 
  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();
  MVT SrcEltVT = SrcVT.getVectorElementType();
 
  assert(DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) &&
         isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
         "Unexpected vector truncate lowering");
 
  MVT ContainerVT = SrcVT;
  if (SrcVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(SrcVT);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }
 
  SDValue Result = Src;
  auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
 
  unsigned NewOpc;
  if (Opc == ISD::TRUNCATE_SSAT_S)
    NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
  else if (Opc == ISD::TRUNCATE_USAT_U)
    NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
  else
    NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
 
  do {
    SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
    MVT ResultVT = ContainerVT.changeVectorElementType(SrcEltVT);
    Result = DAG.getNode(NewOpc, DL, ResultVT, Result, Mask, VL);
  } while (SrcEltVT != DstEltVT);
 
  if (SrcVT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
 
  return Result;
}
 
SDValue
RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(SDValue Op,
                                                    SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op.getOperand(0);
  SDValue Src = Op.getOperand(1);
  MVT VT = Op.getSimpleValueType();
  MVT SrcVT = Src.getSimpleValueType();
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
    ContainerVT =
        SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
 
  // RVV can only widen/truncate fp to types double/half the size as the source.
  if ((VT.getVectorElementType() == MVT::f64 &&
       (SrcVT.getVectorElementType() == MVT::f16 ||
        SrcVT.getVectorElementType() == MVT::bf16)) ||
      ((VT.getVectorElementType() == MVT::f16 ||
        VT.getVectorElementType() == MVT::bf16) &&
       SrcVT.getVectorElementType() == MVT::f64)) {
    // For double rounding, the intermediate rounding should be round-to-odd.
    unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
                                ? RISCVISD::STRICT_FP_EXTEND_VL
                                : RISCVISD::STRICT_VFNCVT_ROD_VL;
    MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
    Src = DAG.getNode(InterConvOpc, DL, DAG.getVTList(InterVT, MVT::Other),
                      Chain, Src, Mask, VL);
    Chain = Src.getValue(1);
  }
 
  unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
                         ? RISCVISD::STRICT_FP_EXTEND_VL
                         : RISCVISD::STRICT_FP_ROUND_VL;
  SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
                            Chain, Src, Mask, VL);
  if (VT.isFixedLengthVector()) {
    // StrictFP operations have two result values. Their lowered result should
    // have same result count.
    SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
    Res = DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
  }
  return Res;
}
 
SDValue
RISCVTargetLowering::lowerVectorFPExtendOrRound(SDValue Op,
                                                SelectionDAG &DAG) const {
  bool IsExtend = Op.getOpcode() == ISD::FP_EXTEND;
  // RVV can only do truncate fp to types half the size as the source. We
  // custom-lower f64->f16 rounds via RVV's round-to-odd float
  // conversion instruction.
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
 
  assert(VT.isVector() && "Unexpected type for vector truncate lowering");
 
  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();
 
  bool IsDirectExtend =
      IsExtend && (VT.getVectorElementType() != MVT::f64 ||
                   (SrcVT.getVectorElementType() != MVT::f16 &&
                    SrcVT.getVectorElementType() != MVT::bf16));
  bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
                                      VT.getVectorElementType() != MVT::bf16) ||
                                     SrcVT.getVectorElementType() != MVT::f64);
 
  bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
 
  // We have regular SD node patterns for direct non-VL extends.
  if (VT.isScalableVector() && IsDirectConv)
    return Op;
 
  // Prepare any fixed-length vector operands.
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
    ContainerVT =
        SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
 
  unsigned ConvOpc = IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::FP_ROUND_VL;
 
  if (IsDirectConv) {
    Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL);
    if (VT.isFixedLengthVector())
      Src = convertFromScalableVector(VT, Src, DAG, Subtarget);
    return Src;
  }
 
  unsigned InterConvOpc =
      IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::VFNCVT_ROD_VL;
 
  MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
  SDValue IntermediateConv =
      DAG.getNode(InterConvOpc, DL, InterVT, Src, Mask, VL);
  SDValue Result =
      DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL);
  if (VT.isFixedLengthVector())
    return convertFromScalableVector(VT, Result, DAG, Subtarget);
  return Result;
}
 
// Given a scalable vector type and an index into it, returns the type for the
// smallest subvector that the index fits in. This can be used to reduce LMUL
// for operations like vslidedown.
//
// E.g. With Zvl128b, index 3 in a nxv4i32 fits within the first nxv2i32.
static std::optional<MVT>
getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
                      const RISCVSubtarget &Subtarget) {
  assert(VecVT.isScalableVector());
  const unsigned EltSize = VecVT.getScalarSizeInBits();
  const unsigned VectorBitsMin = Subtarget.getRealMinVLen();
  const unsigned MinVLMAX = VectorBitsMin / EltSize;
  MVT SmallerVT;
  if (MaxIdx < MinVLMAX)
    SmallerVT = RISCVTargetLowering::getM1VT(VecVT);
  else if (MaxIdx < MinVLMAX * 2)
    SmallerVT =
        RISCVTargetLowering::getM1VT(VecVT).getDoubleNumVectorElementsVT();
  else if (MaxIdx < MinVLMAX * 4)
    SmallerVT = RISCVTargetLowering::getM1VT(VecVT)
                    .getDoubleNumVectorElementsVT()
                    .getDoubleNumVectorElementsVT();
  if (!SmallerVT.isValid() || !VecVT.bitsGT(SmallerVT))
    return std::nullopt;
  return SmallerVT;
}
 
// Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
// first position of a vector, and that vector is slid up to the insert index.
// By limiting the active vector length to index+1 and merging with the
// original vector (with an undisturbed tail policy for elements >= VL), we
// achieve the desired result of leaving all elements untouched except the one
// at VL-1, which is replaced with the desired value.
SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
                                                    SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VecVT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Vec = Op.getOperand(0);
  SDValue Val = Op.getOperand(1);
  MVT ValVT = Val.getSimpleValueType();
  SDValue Idx = Op.getOperand(2);
 
  if (VecVT.getVectorElementType() == MVT::i1) {
    // FIXME: For now we just promote to an i8 vector and insert into that,
    // but this is probably not optimal.
    MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
    Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
    Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
    return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
  }
 
  if ((ValVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
      (ValVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
    // If we don't have vfmv.s.f for f16/bf16, use fmv.x.h first.
    MVT IntVT = VecVT.changeTypeToInteger();
    SDValue IntInsert = DAG.getNode(
        ISD::INSERT_VECTOR_ELT, DL, IntVT, DAG.getBitcast(IntVT, Vec),
        DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Val), Idx);
    return DAG.getBitcast(VecVT, IntInsert);
  }
 
  if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
    auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
    if (!IdxC)
      return SDValue();
 
    unsigned IdxVal = IdxC->getZExtValue();
    unsigned NumElts = VecVT.getVectorNumElements();
    MVT EltVT = VecVT.getVectorElementType();
 
    if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
      unsigned HalfNumElts = NumElts / 2;
      auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
      MVT HalfVT = Lo.getSimpleValueType();
      if (IdxVal < HalfNumElts) {
        SDValue NewLo =
            DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, HalfVT, Lo, Val, Idx);
        return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, NewLo, Hi);
      }
      SDValue NewHi =
          DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, HalfVT, Hi, Val,
                      DAG.getVectorIdxConstant(IdxVal - HalfNumElts, DL));
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, NewHi);
    }
 
    Vec = DAG.getBitcast(XLenVT, Vec);
    SDValue ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
 
    // For 2-element vectors, BUILD_VECTOR is more efficient since it only needs
    // at most 2 instructions.
    if (NumElts == 2) {
      unsigned EltBits = EltVT.getSizeInBits();
      SDValue Elt0, Elt1;
      if (IdxVal == 0) {
        Elt0 = ExtVal;
        Elt1 = DAG.getNode(ISD::SRL, DL, XLenVT, Vec,
                           DAG.getConstant(EltBits, DL, XLenVT));
      } else {
        Elt0 = Vec;
        Elt1 = ExtVal;
      }
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, Elt0, Elt1);
    }
 
    // For 4/8-element vectors, use MVM(or MERGE) instruction which does bitwise
    // select: rd = (~mask & rd) | (mask & rs1).
    // This generates: slli + lui/li + mvm
    if (NumElts == 4 || NumElts == 8) {
      unsigned EltBits = EltVT.getSizeInBits();
      unsigned ShiftAmt = IdxVal * EltBits;
      uint64_t PosMask = ((1ULL << EltBits) - 1) << ShiftAmt;
 
      SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, XLenVT, ExtVal,
                                       DAG.getConstant(ShiftAmt, DL, XLenVT));
      SDValue Mask = DAG.getConstant(PosMask, DL, XLenVT);
      SDValue Result =
          DAG.getNode(RISCVISD::MERGE, DL, XLenVT, Mask, Vec, ShiftedVal);
      return DAG.getBitcast(VecVT, Result);
    }
 
    return SDValue();
  }
 
  MVT ContainerVT = VecVT;
  // If the operand is a fixed-length vector, convert to a scalable one.
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
 
  // If we know the index we're going to insert at, we can shrink Vec so that
  // we're performing the scalar inserts and slideup on a smaller LMUL.
  SDValue OrigVec = Vec;
  std::optional<unsigned> AlignedIdx;
  if (auto *IdxC = dyn_cast<ConstantSDNode>(Idx)) {
    const unsigned OrigIdx = IdxC->getZExtValue();
    // Do we know an upper bound on LMUL?
    if (auto ShrunkVT = getSmallestVTForIndex(ContainerVT, OrigIdx,
                                              DL, DAG, Subtarget)) {
      ContainerVT = *ShrunkVT;
      AlignedIdx = 0;
    }
 
    // If we're compiling for an exact VLEN value, we can always perform
    // the insert in m1 as we can determine the register corresponding to
    // the index in the register group.
    const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
    if (auto VLEN = Subtarget.getRealVLen(); VLEN && ContainerVT.bitsGT(M1VT)) {
      EVT ElemVT = VecVT.getVectorElementType();
      unsigned ElemsPerVReg = *VLEN / ElemVT.getFixedSizeInBits();
      unsigned RemIdx = OrigIdx % ElemsPerVReg;
      unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
      AlignedIdx = SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
      Idx = DAG.getVectorIdxConstant(RemIdx, DL);
      ContainerVT = M1VT;
    }
 
    if (AlignedIdx)
      Vec = DAG.getExtractSubvector(DL, ContainerVT, Vec, *AlignedIdx);
  }
 
  bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
  // Even i64-element vectors on RV32 can be lowered without scalar
  // legalization if the most-significant 32 bits of the value are not affected
  // by the sign-extension of the lower 32 bits. This applies to i32 constants
  // and sign_extend of i32 values.
  if (!IsLegalInsert) {
    if (isa<ConstantSDNode>(Val)) {
      const auto *CVal = cast<ConstantSDNode>(Val);
      if (isInt<32>(CVal->getSExtValue())) {
        IsLegalInsert = true;
        Val = DAG.getSignedConstant(CVal->getSExtValue(), DL, MVT::i32);
      }
    } else if (Val.getOpcode() == ISD::SIGN_EXTEND &&
               Val.getOperand(0).getValueType() == MVT::i32) {
      IsLegalInsert = true;
      Val = Val.getOperand(0);
    }
  }
 
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
  SDValue ValInVec;
 
  if (IsLegalInsert) {
    unsigned Opc =
        VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
    if (isNullConstant(Idx)) {
      if (!VecVT.isFloatingPoint())
        Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
      Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
 
      if (AlignedIdx)
        Vec = DAG.getInsertSubvector(DL, OrigVec, Vec, *AlignedIdx);
      if (!VecVT.isFixedLengthVector())
        return Vec;
      return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
    }
 
    ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget);
  } else {
    // On RV32, i64-element vectors must be specially handled to place the
    // value at element 0, by using two vslide1down instructions in sequence on
    // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
    // this.
    SDValue ValLo, ValHi;
    std::tie(ValLo, ValHi) = DAG.SplitScalar(Val, DL, MVT::i32, MVT::i32);
    MVT I32ContainerVT =
        MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
    SDValue I32Mask =
        getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
    // Limit the active VL to two.
    SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
    // If the Idx is 0 we can insert directly into the vector.
    if (isNullConstant(Idx)) {
      // First slide in the lo value, then the hi in above it. We use slide1down
      // to avoid the register group overlap constraint of vslide1up.
      ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
                             Vec, Vec, ValLo, I32Mask, InsertI64VL);
      // If the source vector is undef don't pass along the tail elements from
      // the previous slide1down.
      SDValue Tail = Vec.isUndef() ? Vec : ValInVec;
      ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
                             Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
      // Bitcast back to the right container type.
      ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
 
      if (AlignedIdx)
        ValInVec = DAG.getInsertSubvector(DL, OrigVec, ValInVec, *AlignedIdx);
      if (!VecVT.isFixedLengthVector())
        return ValInVec;
      return convertFromScalableVector(VecVT, ValInVec, DAG, Subtarget);
    }
 
    // First slide in the lo value, then the hi in above it. We use slide1down
    // to avoid the register group overlap constraint of vslide1up.
    ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
                           DAG.getUNDEF(I32ContainerVT),
                           DAG.getUNDEF(I32ContainerVT), ValLo,
                           I32Mask, InsertI64VL);
    ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
                           DAG.getUNDEF(I32ContainerVT), ValInVec, ValHi,
                           I32Mask, InsertI64VL);
    // Bitcast back to the right container type.
    ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
  }
 
  // Now that the value is in a vector, slide it into position.
  SDValue InsertVL =
      DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
 
  // Use tail agnostic policy if Idx is the last index of Vec.
  unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
  if (VecVT.isFixedLengthVector() && isa<ConstantSDNode>(Idx) &&
      Idx->getAsZExtVal() + 1 == VecVT.getVectorNumElements())
    Policy = RISCVVType::TAIL_AGNOSTIC;
  SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec,
                                Idx, Mask, InsertVL, Policy);
 
  if (AlignedIdx)
    Slideup = DAG.getInsertSubvector(DL, OrigVec, Slideup, *AlignedIdx);
  if (!VecVT.isFixedLengthVector())
    return Slideup;
  return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
}
 
// Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
// extract the first element: (extractelt (slidedown vec, idx), 0). For integer
// types this is done using VMV_X_S to allow us to glean information about the
// sign bits of the result.
SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                     SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Idx = Op.getOperand(1);
  SDValue Vec = Op.getOperand(0);
  EVT EltVT = Op.getValueType();
  MVT VecVT = Vec.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  if (VecVT.getVectorElementType() == MVT::i1) {
    // Use vfirst.m to extract the first bit.
    if (isNullConstant(Idx)) {
      MVT ContainerVT = VecVT;
      if (VecVT.isFixedLengthVector()) {
        ContainerVT = getContainerForFixedLengthVector(VecVT);
        Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
      }
      auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
      SDValue Vfirst =
          DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL);
      SDValue Res = DAG.getSetCC(DL, XLenVT, Vfirst,
                                 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
      return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
    }
    if (VecVT.isFixedLengthVector()) {
      unsigned NumElts = VecVT.getVectorNumElements();
      if (NumElts >= 8) {
        MVT WideEltVT;
        unsigned WidenVecLen;
        SDValue ExtractElementIdx;
        SDValue ExtractBitIdx;
        unsigned MaxEEW = Subtarget.getELen();
        MVT LargestEltVT = MVT::getIntegerVT(
            std::min(MaxEEW, unsigned(XLenVT.getSizeInBits())));
        if (NumElts <= LargestEltVT.getSizeInBits()) {
          assert(isPowerOf2_32(NumElts) &&
                 "the number of elements should be power of 2");
          WideEltVT = MVT::getIntegerVT(NumElts);
          WidenVecLen = 1;
          ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
          ExtractBitIdx = Idx;
        } else {
          WideEltVT = LargestEltVT;
          WidenVecLen = NumElts / WideEltVT.getSizeInBits();
          // extract element index = index / element width
          ExtractElementIdx = DAG.getNode(
              ISD::SRL, DL, XLenVT, Idx,
              DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
          // mask bit index = index % element width
          ExtractBitIdx = DAG.getNode(
              ISD::AND, DL, XLenVT, Idx,
              DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
        }
        MVT WideVT = MVT::getVectorVT(WideEltVT, WidenVecLen);
        Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
        SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
                                         Vec, ExtractElementIdx);
        // Extract the bit from GPR.
        SDValue ShiftRight =
            DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
        SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
                                  DAG.getConstant(1, DL, XLenVT));
        return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
      }
    }
    // Otherwise, promote to an i8 vector and extract from that.
    MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
    Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
  }
 
  if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
      (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
    // If we don't have vfmv.f.s for f16/bf16, extract to a gpr then use fmv.h.x
    MVT IntVT = VecVT.changeTypeToInteger();
    SDValue IntVec = DAG.getBitcast(IntVT, Vec);
    SDValue IntExtract =
        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT, IntVec, Idx);
    return DAG.getNode(RISCVISD::FMV_H_X, DL, EltVT, IntExtract);
  }
 
  if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
    if (VecVT != MVT::v4i16 && VecVT != MVT::v2i16 && VecVT != MVT::v8i8 &&
        VecVT != MVT::v4i8 && VecVT != MVT::v2i32)
      return SDValue();
 
    if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
      auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
      if (!IdxC)
        return SDValue();
      unsigned IdxVal = IdxC->getZExtValue();
      unsigned HalfNumElts = VecVT.getVectorNumElements() / 2;
      auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
      if (IdxVal < HalfNumElts)
        return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Lo, Idx);
      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Hi,
                         DAG.getVectorIdxConstant(IdxVal - HalfNumElts, DL));
    }
 
    SDValue Extracted = DAG.getBitcast(XLenVT, Vec);
    unsigned ElemWidth = VecVT.getVectorElementType().getSizeInBits();
    SDValue Shamt = DAG.getNode(ISD::MUL, DL, XLenVT, Idx,
                                DAG.getConstant(ElemWidth, DL, XLenVT));
    return DAG.getNode(ISD::SRL, DL, XLenVT, Extracted, Shamt);
  }
 
  // If this is a fixed vector, we need to convert it to a scalable vector.
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
 
  // If we're compiling for an exact VLEN value and we have a known
  // constant index, we can always perform the extract in m1 (or
  // smaller) as we can determine the register corresponding to
  // the index in the register group.
  const auto VLen = Subtarget.getRealVLen();
  if (auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
      IdxC && VLen && VecVT.getSizeInBits().getKnownMinValue() > *VLen) {
    MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
    unsigned OrigIdx = IdxC->getZExtValue();
    EVT ElemVT = VecVT.getVectorElementType();
    unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
    unsigned RemIdx = OrigIdx % ElemsPerVReg;
    unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
    unsigned ExtractIdx =
        SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
    Vec = DAG.getExtractSubvector(DL, M1VT, Vec, ExtractIdx);
    Idx = DAG.getVectorIdxConstant(RemIdx, DL);
    ContainerVT = M1VT;
  }
 
  // Reduce the LMUL of our slidedown and vmv.x.s to the smallest LMUL which
  // contains our index.
  std::optional<uint64_t> MaxIdx;
  if (VecVT.isFixedLengthVector())
    MaxIdx = VecVT.getVectorNumElements() - 1;
  if (auto *IdxC = dyn_cast<ConstantSDNode>(Idx))
    MaxIdx = IdxC->getZExtValue();
  if (MaxIdx) {
    if (auto SmallerVT =
            getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) {
      ContainerVT = *SmallerVT;
      Vec = DAG.getExtractSubvector(DL, ContainerVT, Vec, 0);
    }
  }
 
  // If after narrowing, the required slide is still greater than LMUL2,
  // fallback to generic expansion and go through the stack.  This is done
  // for a subtle reason: extracting *all* elements out of a vector is
  // widely expected to be linear in vector size, but because vslidedown
  // is linear in LMUL, performing N extracts using vslidedown becomes
  // O(n^2) / (VLEN/ETYPE) work.  On the surface, going through the stack
  // seems to have the same problem (the store is linear in LMUL), but the
  // generic expansion *memoizes* the store, and thus for many extracts of
  // the same vector we end up with one store and a bunch of loads.
  // TODO: We don't have the same code for insert_vector_elt because we
  // have BUILD_VECTOR and handle the degenerate case there.  Should we
  // consider adding an inverse BUILD_VECTOR node?
  MVT LMUL2VT =
      RISCVTargetLowering::getM1VT(ContainerVT).getDoubleNumVectorElementsVT();
  if (ContainerVT.bitsGT(LMUL2VT) && VecVT.isFixedLengthVector())
    return SDValue();
 
  // If the index is 0, the vector is already in the right position.
  if (!isNullConstant(Idx)) {
    // Use a VL of 1 to avoid processing more elements than we need.
    auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
    Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
                        DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
  }
 
  if (!EltVT.isInteger()) {
    // Floating-point extracts are handled in TableGen.
    return DAG.getExtractVectorElt(DL, EltVT, Vec, 0);
  }
 
  SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
  return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
}
 
// Some RVV intrinsics may claim that they want an integer operand to be
// promoted or expanded.
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  assert((Op.getOpcode() == ISD::INTRINSIC_VOID ||
          Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
         "Unexpected opcode");
 
  if (!Subtarget.hasVInstructions())
    return SDValue();
 
  bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
                  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
  unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
 
  SDLoc DL(Op);
 
  const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
      RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
  if (!II || !II->hasScalarOperand())
    return SDValue();
 
  unsigned SplatOp = II->ScalarOperand + 1 + HasChain;
  assert(SplatOp < Op.getNumOperands());
 
  SmallVector<SDValue, 8> Operands(Op->ops());
  SDValue &ScalarOp = Operands[SplatOp];
  MVT OpVT = ScalarOp.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  // If this isn't a scalar, or its type is XLenVT we're done.
  if (!OpVT.isScalarInteger() || OpVT == XLenVT)
    return SDValue();
 
  // Simplest case is that the operand needs to be promoted to XLenVT.
  if (OpVT.bitsLT(XLenVT)) {
    // If the operand is a constant, sign extend to increase our chances
    // of being able to use a .vi instruction. ANY_EXTEND would become a
    // a zero extend and the simm5 check in isel would fail.
    // FIXME: Should we ignore the upper bits in isel instead?
    unsigned ExtOpc =
        isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
    ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
    return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
  }
 
  // Use the previous operand to get the vXi64 VT. The result might be a mask
  // VT for compares. Using the previous operand assumes that the previous
  // operand will never have a smaller element size than a scalar operand and
  // that a widening operation never uses SEW=64.
  // NOTE: If this fails the below assert, we can probably just find the
  // element count from any operand or result and use it to construct the VT.
  assert(II->ScalarOperand > 0 && "Unexpected splat operand!");
  MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
 
  // The more complex case is when the scalar is larger than XLenVT.
  assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
         VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
 
  // If this is a sign-extended 32-bit value, we can truncate it and rely on the
  // instruction to sign-extend since SEW>XLEN.
  if (DAG.ComputeNumSignBits(ScalarOp) > 32) {
    ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp);
    return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
  }
 
  switch (IntNo) {
  case Intrinsic::riscv_vslide1up:
  case Intrinsic::riscv_vslide1down:
  case Intrinsic::riscv_vslide1up_mask:
  case Intrinsic::riscv_vslide1down_mask: {
    // We need to special case these when the scalar is larger than XLen.
    unsigned NumOps = Op.getNumOperands();
    bool IsMasked = NumOps == 7;
 
    // Convert the vector source to the equivalent nxvXi32 vector.
    MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
    SDValue Vec = DAG.getBitcast(I32VT, Operands[2]);
    SDValue ScalarLo, ScalarHi;
    std::tie(ScalarLo, ScalarHi) =
        DAG.SplitScalar(ScalarOp, DL, MVT::i32, MVT::i32);
 
    // Double the VL since we halved SEW.
    SDValue AVL = getVLOperand(Op);
    SDValue I32VL;
 
    // Optimize for constant AVL
    if (isa<ConstantSDNode>(AVL)) {
      const auto [MinVLMAX, MaxVLMAX] =
          RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget);
 
      uint64_t AVLInt = AVL->getAsZExtVal();
      if (AVLInt <= MinVLMAX) {
        I32VL = DAG.getConstant(2 * AVLInt, DL, XLenVT);
      } else if (AVLInt >= 2 * MaxVLMAX) {
        // Just set vl to VLMAX in this situation
        I32VL = DAG.getRegister(RISCV::X0, XLenVT);
      } else {
        // For AVL between (MinVLMAX, 2 * MaxVLMAX), the actual working vl
        // is related to the hardware implementation.
        // So let the following code handle
      }
    }
    if (!I32VL) {
      RISCVVType::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT);
      SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT);
      unsigned Sew = RISCVVType::encodeSEW(VT.getScalarSizeInBits());
      SDValue SEW = DAG.getConstant(Sew, DL, XLenVT);
      SDValue SETVL =
          DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, MVT::i32);
      // Using vsetvli instruction to get actually used length which related to
      // the hardware implementation
      SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL,
                               SEW, LMUL);
      I32VL =
          DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
    }
 
    SDValue I32Mask = getAllOnesMask(I32VT, I32VL, DL, DAG);
 
    // Shift the two scalar parts in using SEW=32 slide1up/slide1down
    // instructions.
    SDValue Passthru;
    if (IsMasked)
      Passthru = DAG.getUNDEF(I32VT);
    else
      Passthru = DAG.getBitcast(I32VT, Operands[1]);
 
    if (IntNo == Intrinsic::riscv_vslide1up ||
        IntNo == Intrinsic::riscv_vslide1up_mask) {
      Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
                        ScalarHi, I32Mask, I32VL);
      Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
                        ScalarLo, I32Mask, I32VL);
    } else {
      Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
                        ScalarLo, I32Mask, I32VL);
      Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
                        ScalarHi, I32Mask, I32VL);
    }
 
    // Convert back to nxvXi64.
    Vec = DAG.getBitcast(VT, Vec);
 
    if (!IsMasked)
      return Vec;
    // Apply mask after the operation.
    SDValue Mask = Operands[NumOps - 3];
    SDValue MaskedOff = Operands[1];
    // Assume Policy operand is the last operand.
    uint64_t Policy = Operands[NumOps - 1]->getAsZExtVal();
    // We don't need to select maskedoff if it's undef.
    if (MaskedOff.isUndef())
      return Vec;
    // TAMU
    if (Policy == RISCVVType::TAIL_AGNOSTIC)
      return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
                         DAG.getUNDEF(VT), AVL);
    // TUMA or TUMU: Currently we always emit tumu policy regardless of tuma.
    // It's fine because vmerge does not care mask policy.
    return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
                       MaskedOff, AVL);
  }
  }
 
  // We need to convert the scalar to a splat vector.
  SDValue VL = getVLOperand(Op);
  assert(VL.getValueType() == XLenVT);
  ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG);
  return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
}
 
// Lower the llvm.get.vector.length intrinsic to vsetvli. We only support
// scalable vector llvm.get.vector.length for now.
//
// We need to convert from a scalable VF to a vsetvli with VLMax equal to
// (vscale * VF). The vscale and VF are independent of element width. We use
// SEW=8 for the vsetvli because it is the only element width that supports all
// fractional LMULs. The LMUL is chosen so that with SEW=8 the VLMax is
// (vscale * VF). Where vscale is defined as VLEN/RVVBitsPerBlock. The
// InsertVSETVLI pass can fix up the vtype of the vsetvli if a different
// SEW and LMUL are better for the surrounding vector instructions.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
  MVT XLenVT = Subtarget.getXLenVT();
 
  // The smallest LMUL is only valid for the smallest element width.
  const unsigned ElementWidth = 8;
 
  // Determine the VF that corresponds to LMUL 1 for ElementWidth.
  unsigned LMul1VF = RISCV::RVVBitsPerBlock / ElementWidth;
  // We don't support VF==1 with ELEN==32.
  [[maybe_unused]] unsigned MinVF =
      RISCV::RVVBitsPerBlock / Subtarget.getELen();
 
  [[maybe_unused]] unsigned VF = N->getConstantOperandVal(2);
  assert(VF >= MinVF && VF <= (LMul1VF * 8) && isPowerOf2_32(VF) &&
         "Unexpected VF");
 
  bool Fractional = VF < LMul1VF;
  unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
  unsigned VLMUL = (unsigned)RISCVVType::encodeLMUL(LMulVal, Fractional);
  unsigned VSEW = RISCVVType::encodeSEW(ElementWidth);
 
  SDLoc DL(N);
 
  SDValue LMul = DAG.getTargetConstant(VLMUL, DL, XLenVT);
  SDValue Sew = DAG.getTargetConstant(VSEW, DL, XLenVT);
 
  SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1));
 
  SDValue ID = DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, XLenVT);
  SDValue Res =
      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul);
  return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
}
 
static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget) {
  SDValue Op0 = Op.getOperand(0);
  MVT OpVT = Op0.getSimpleValueType();
  MVT ContainerVT = OpVT;
  if (OpVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(OpVT, Subtarget);
    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
  }
  MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
  SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Op0, Mask, VL);
  if (Op.getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
    return Res;
 
  // Convert -1 to VL.
  SDValue Setcc =
      DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
  VL = DAG.getElementCount(DL, XLenVT, OpVT.getVectorElementCount());
  return DAG.getSelect(DL, XLenVT, Setcc, VL, Res);
}
 
static SDValue lowerCONVERT_FROM_ARBITRARY_FP(SDValue Op, SelectionDAG &DAG,
                                              const RISCVSubtarget &Subtarget) {
  assert(Op.getOpcode() == ISD::CONVERT_FROM_ARBITRARY_FP);
  const uint64_t SemEnum = Op->getConstantOperandVal(1);
  const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
 
  if (Sem != APFloatBase::S_Float8E5M2)
    return SDValue();
 
  SDValue Src = Op.getOperand(0);
  EVT SrcEVT = Src.getValueType();
  if (!SrcEVT.isSimple() || SrcEVT.getVectorElementType() != MVT::i8)
    return SDValue();
  MVT SrcVT = SrcEVT.getSimpleVT();
  MVT DstVT = Op.getSimpleValueType();
  assert(DstVT.getVectorElementType() == MVT::bf16);
  SDLoc DL(Op);
 
  MVT SrcContainerVT = SrcVT;
  MVT DstContainerVT = DstVT;
  if (SrcVT.isFixedLengthVector()) {
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
    DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
  }
  SDValue VL =
      getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget).second;
 
  SDValue Ops[] = {DAG.getTargetConstant(Intrinsic::riscv_vfwcvt_f_f_v_alt, DL,
                                         Subtarget.getXLenVT()),
                   DAG.getPOISON(DstContainerVT), // Passthru
                   Src, VL};
  SDValue NewVal =
      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, DstContainerVT, Ops);
  if (SrcVT.isFixedLengthVector())
    NewVal = convertFromScalableVector(DstVT, NewVal, DAG, Subtarget);
  return NewVal;
}
 
static inline void promoteVCIXScalar(SDValue Op,
                                     MutableArrayRef<SDValue> Operands,
                                     SelectionDAG &DAG) {
  const RISCVSubtarget &Subtarget =
      DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
 
  bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
                  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
  unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
  SDLoc DL(Op);
 
  const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
      RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
  if (!II || !II->hasScalarOperand())
    return;
 
  unsigned SplatOp = II->ScalarOperand + 1;
  assert(SplatOp < Op.getNumOperands());
 
  SDValue &ScalarOp = Operands[SplatOp];
  MVT OpVT = ScalarOp.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  // The code below is partially copied from lowerVectorIntrinsicScalars.
  // If this isn't a scalar, or its type is XLenVT we're done.
  if (!OpVT.isScalarInteger() || OpVT == XLenVT)
    return;
 
  // Manually emit promote operation for scalar operation.
  if (OpVT.bitsLT(XLenVT)) {
    unsigned ExtOpc =
        isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
    ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
  }
}
 
static void processVCIXOperands(SDValue OrigOp,
                                MutableArrayRef<SDValue> Operands,
                                SelectionDAG &DAG) {
  promoteVCIXScalar(OrigOp, Operands, DAG);
  const RISCVSubtarget &Subtarget =
      DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
  for (SDValue &V : Operands) {
    EVT ValType = V.getValueType();
    if (ValType.isVector() && ValType.isFloatingPoint()) {
      MVT InterimIVT =
          MVT::getVectorVT(MVT::getIntegerVT(ValType.getScalarSizeInBits()),
                           ValType.getVectorElementCount());
      V = DAG.getBitcast(InterimIVT, V);
    }
    if (ValType.isFixedLengthVector()) {
      MVT OpContainerVT =
          getContainerForFixedLengthVector(V.getSimpleValueType(), Subtarget);
      V = convertToScalableVector(OpContainerVT, V, DAG, Subtarget);
    }
  }
}
 
// LMUL * VLEN should be greater than or equal to EGS * SEW
static inline bool isValidEGW(int EGS, EVT VT,
                              const RISCVSubtarget &Subtarget) {
  return (Subtarget.getRealMinVLen() *
             VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
         EGS * VT.getScalarSizeInBits();
}
 
static unsigned getRVPShiftOpcode(Intrinsic::ID IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable(
        "Unexpected RISC-V packed saturating and rounding shift intrinsic");
  case Intrinsic::riscv_psll:
    return RISCVISD::PSLL;
  case Intrinsic::riscv_psrl:
    return RISCVISD::PSRL;
  case Intrinsic::riscv_psra:
    return RISCVISD::PSRA;
  case Intrinsic::riscv_pssha:
    return RISCVISD::PSSHA;
  case Intrinsic::riscv_psshar:
    return RISCVISD::PSSHAR;
  case Intrinsic::riscv_psshl:
    return RISCVISD::PSSHL;
  case Intrinsic::riscv_psshlr:
    return RISCVISD::PSSHLR;
  }
}
 
static unsigned getRVPNarrowingShiftOpcode(Intrinsic::ID IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed narrowing shift intrinsic");
  case Intrinsic::riscv_pnsrl:
    return RISCVISD::PNSRL;
  case Intrinsic::riscv_pnsra:
    return RISCVISD::PNSRA;
  case Intrinsic::riscv_pnsrar:
    return RISCVISD::PNSRAR;
  }
}
 
static SDValue lowerPZExt(SDValue Src, const SDLoc &DL, SelectionDAG &DAG,
                          const RISCVSubtarget &Subtarget) {
  MVT VT = Src.getSimpleValueType();
  MVT PPairVT =
      MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits() / 2),
                       VT.getVectorNumElements() * 2);
  Src = DAG.getBitcast(PPairVT, Src);
  SDValue Zero = DAG.getNode(ISD::SPLAT_VECTOR, DL, PPairVT,
                             DAG.getConstant(0, DL, Subtarget.getXLenVT()));
  SDValue Res = DAG.getNode(RISCVISD::PPAIRE, DL, PPairVT, Src, Zero);
  return DAG.getBitcast(VT, Res);
}
 
static unsigned getRVPMulHighOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed multiply high intrinsic");
  case Intrinsic::riscv_pmulhr:
    return RISCVISD::MULHR;
  case Intrinsic::riscv_pmulhru:
    return RISCVISD::MULHRU;
  case Intrinsic::riscv_pmulhsu:
    return RISCVISD::MULHSU;
  case Intrinsic::riscv_pmulhrsu:
    return RISCVISD::MULHRSU;
  }
}
 
static unsigned getRVScalarMulHighOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V scalar multiply high intrinsic");
  case Intrinsic::riscv_mulhr_i32:
    return RISCVISD::MULHR;
  case Intrinsic::riscv_mulhru_u32:
    return RISCVISD::MULHRU;
  case Intrinsic::riscv_mulhsu_i32:
    return RISCVISD::MULHSU;
  case Intrinsic::riscv_mulhrsu_i32:
    return RISCVISD::MULHRSU;
  }
}
 
static unsigned getRVPMulHighAccumulateOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable(
        "Unexpected RISC-V packed multiply high accumulate intrinsic");
  case Intrinsic::riscv_pmhacc:
    return RISCVISD::MHACC;
  case Intrinsic::riscv_pmhracc:
    return RISCVISD::MHRACC;
  case Intrinsic::riscv_pmhaccu:
    return RISCVISD::MHACCU;
  case Intrinsic::riscv_pmhraccu:
    return RISCVISD::MHRACCU;
  case Intrinsic::riscv_pmhaccsu:
    return RISCVISD::MHACCSU;
  case Intrinsic::riscv_pmhraccsu:
    return RISCVISD::MHRACCSU;
  }
}
 
/// Return the multiply high accumulate node for \p IntNo.
static unsigned getRVPMulHighAccumulateByHalvesOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed multiply high accumulate by "
                     "halves intrinsic");
  case Intrinsic::riscv_pmhacc_b0:
    return RISCVISD::MHACC_H_B0;
  case Intrinsic::riscv_pmhacc_b1:
    return RISCVISD::MHACC_H_B1;
  case Intrinsic::riscv_pmhaccsu_b0:
    return RISCVISD::MHACCSU_H_B0;
  case Intrinsic::riscv_pmhaccsu_b1:
    return RISCVISD::MHACCSU_H_B1;
  case Intrinsic::riscv_pmhacc_h0:
    return RISCVISD::MHACC_W_H0;
  case Intrinsic::riscv_pmhacc_h1:
    return RISCVISD::MHACC_W_H1;
  case Intrinsic::riscv_pmhaccsu_h0:
    return RISCVISD::MHACCSU_W_H0;
  case Intrinsic::riscv_pmhaccsu_h1:
    return RISCVISD::MHACCSU_W_H1;
  }
}
 
/// Return the multiply-high-parts node for a multiply-high-parts intrinsic.
/// The scalar spelling maps to the same node; its product is the first
/// element.
static unsigned getRVPMulHighPartsOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V multiply-high-parts intrinsic");
  case Intrinsic::riscv_pmulh_b0:
    return RISCVISD::PMULH_H_B0;
  case Intrinsic::riscv_pmulh_b1:
    return RISCVISD::PMULH_H_B1;
  case Intrinsic::riscv_pmulhsu_b0:
    return RISCVISD::PMULHSU_H_B0;
  case Intrinsic::riscv_pmulhsu_b1:
    return RISCVISD::PMULHSU_H_B1;
  }
}
 
/// Return the word multiply-high-parts node for a word multiply-high-parts
/// intrinsic. The scalar spelling maps to the same node; its product is the
/// first element.
static unsigned getRVPMulHighPartsWOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V word multiply-high-parts intrinsic");
  case Intrinsic::riscv_pmulh_h0:
  case Intrinsic::riscv_mulh_h0:
    return RISCVISD::PMULH_W_H0;
  case Intrinsic::riscv_pmulh_h1:
  case Intrinsic::riscv_mulh_h1:
    return RISCVISD::PMULH_W_H1;
  case Intrinsic::riscv_pmulhsu_h0:
  case Intrinsic::riscv_mulhsu_h0:
    return RISCVISD::PMULHSU_W_H0;
  case Intrinsic::riscv_pmulhsu_h1:
  case Intrinsic::riscv_mulhsu_h1:
    return RISCVISD::PMULHSU_W_H1;
  }
}
 
/// Return the scalar halfword multiply-high-parts node for \p IntNo (RV32
/// mulh.h0/h1). The packed word spelling maps to the same node; its product
/// is the first element.
static unsigned getRVPScalarMulHighPartsOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V scalar multiply-high-parts intrinsic");
  case Intrinsic::riscv_pmulh_h0:
  case Intrinsic::riscv_mulh_h0:
    return RISCVISD::MULH_H0;
  case Intrinsic::riscv_pmulh_h1:
  case Intrinsic::riscv_mulh_h1:
    return RISCVISD::MULH_H1;
  case Intrinsic::riscv_pmulhsu_h0:
  case Intrinsic::riscv_mulhsu_h0:
    return RISCVISD::MULHSU_H0;
  case Intrinsic::riscv_pmulhsu_h1:
  case Intrinsic::riscv_mulhsu_h1:
    return RISCVISD::MULHSU_H1;
  }
}
 
static unsigned getRVPQFormatAccScalarOpcode(Intrinsic::ID IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
  case Intrinsic::riscv_mqacc_00:
  case Intrinsic::riscv_pmqacc_h00:
    return RISCVISD::MQACC_00;
  case Intrinsic::riscv_mqacc_01:
  case Intrinsic::riscv_pmqacc_h01:
    return RISCVISD::MQACC_01;
  case Intrinsic::riscv_mqacc_11:
  case Intrinsic::riscv_pmqacc_h11:
    return RISCVISD::MQACC_11;
  case Intrinsic::riscv_mqracc_00:
  case Intrinsic::riscv_pmqracc_h00:
    return RISCVISD::MQRACC_00;
  case Intrinsic::riscv_mqracc_01:
  case Intrinsic::riscv_pmqracc_h01:
    return RISCVISD::MQRACC_01;
  case Intrinsic::riscv_mqracc_11:
  case Intrinsic::riscv_pmqracc_h11:
    return RISCVISD::MQRACC_11;
  }
}
 
static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
  case Intrinsic::riscv_mqacc_00:
  case Intrinsic::riscv_pmqacc_h00:
    return RISCVISD::PMQACC_W_H00;
  case Intrinsic::riscv_mqacc_01:
  case Intrinsic::riscv_pmqacc_h01:
    return RISCVISD::PMQACC_W_H01;
  case Intrinsic::riscv_mqacc_11:
  case Intrinsic::riscv_pmqacc_h11:
    return RISCVISD::PMQACC_W_H11;
  case Intrinsic::riscv_mqracc_00:
  case Intrinsic::riscv_pmqracc_h00:
    return RISCVISD::PMQRACC_W_H00;
  case Intrinsic::riscv_mqracc_01:
  case Intrinsic::riscv_pmqracc_h01:
    return RISCVISD::PMQRACC_W_H01;
  case Intrinsic::riscv_mqracc_11:
  case Intrinsic::riscv_pmqracc_h11:
    return RISCVISD::PMQRACC_W_H11;
  }
}
 
static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed horizontal multiply intrinsic");
  case Intrinsic::riscv_pm4add:
    return RISCVISD::PM4ADD;
  case Intrinsic::riscv_pm2add:
    return RISCVISD::PM2ADD;
  case Intrinsic::riscv_pm2add_x:
    return RISCVISD::PM2ADD_X;
  case Intrinsic::riscv_pm4addu:
    return RISCVISD::PM4ADDU;
  case Intrinsic::riscv_pm2addu:
    return RISCVISD::PM2ADDU;
  case Intrinsic::riscv_pmq2add:
    return RISCVISD::PMQ2ADD;
  case Intrinsic::riscv_pmqr2add:
    return RISCVISD::PMQR2ADD;
  case Intrinsic::riscv_pm2sadd:
    return RISCVISD::PM2SADD;
  case Intrinsic::riscv_pm2sadd_x:
    return RISCVISD::PM2SADD_X;
  case Intrinsic::riscv_pm2sub:
    return RISCVISD::PM2SUB;
  case Intrinsic::riscv_pm2sub_x:
    return RISCVISD::PM2SUB_X;
  case Intrinsic::riscv_pm4addsu:
    return RISCVISD::PM4ADDSU;
  case Intrinsic::riscv_pm2addsu:
    return RISCVISD::PM2ADDSU;
  }
}
 
static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
                                        SDValue Rs2, const SDLoc &DL,
                                        SelectionDAG &DAG) {
  auto Extract = [&](SDValue V, unsigned Idx) {
    return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
  };
 
  if (Rs1.getSimpleValueType() == MVT::v4i16) {
    auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
    auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
    unsigned MulOpc, AccOpc;
    switch (IntNo) {
    default:
      llvm_unreachable("Unexpected RV32 horizontal multiply intrinsic");
    case Intrinsic::riscv_pm4add:
      MulOpc = RISCVISD::PM2WADD;
      AccOpc = RISCVISD::PM2WADDA;
      break;
    case Intrinsic::riscv_pm4addu:
      MulOpc = RISCVISD::PM2WADDU;
      AccOpc = RISCVISD::PM2WADDAU;
      break;
    case Intrinsic::riscv_pm4addsu:
      MulOpc = RISCVISD::PM2WADDSU;
      AccOpc = RISCVISD::PM2WADDASU;
      break;
    }
    SDValue Acc = DAG.getNode(MulOpc, DL, MVT::v2i32, Rs1Lo, Rs2Lo);
    Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
    return DAG.getBitcast(MVT::i64, Acc);
  }
 
  assert(Rs1.getSimpleValueType() == MVT::v2i32 &&
         "Unexpected RV32 horizontal multiply source type");
  SDValue Rs1Lo = Extract(Rs1, 0);
  SDValue Rs1Hi = Extract(Rs1, 1);
  SDValue Rs2Lo = Extract(Rs2, 0);
  SDValue Rs2Hi = Extract(Rs2, 1);
 
  if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) {
    unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC
                                                        : RISCVISD::MQRWACC;
    SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
    SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
    SDValue Acc = DAG.getNode(AccOpc, DL, VTs, {Zero, Zero, Rs1Lo, Rs2Lo});
    Acc = DAG.getNode(AccOpc, DL, VTs, {Acc, Acc.getValue(1), Rs1Hi, Rs2Hi});
    return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Acc, Acc.getValue(1));
  }
 
  unsigned MulOpc = ISD::SMUL_LOHI;
  if (IntNo == Intrinsic::riscv_pm2addu)
    MulOpc = ISD::UMUL_LOHI;
  else if (IntNo == Intrinsic::riscv_pm2addsu)
    MulOpc = RISCVISD::WMULSU;
  bool IsSub =
      IntNo == Intrinsic::riscv_pm2sub || IntNo == Intrinsic::riscv_pm2sub_x;
  if (IntNo == Intrinsic::riscv_pm2add_x || IntNo == Intrinsic::riscv_pm2sub_x)
    std::swap(Rs2Lo, Rs2Hi);
 
  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
  SDValue LoMul = DAG.getNode(MulOpc, DL, VTs, Rs1Lo, Rs2Lo);
  SDValue HiMul = DAG.getNode(MulOpc, DL, VTs, Rs1Hi, Rs2Hi);
  unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD;
  SDValue Res = DAG.getNode(Opc, DL, VTs, LoMul, LoMul.getValue(1), HiMul,
                            HiMul.getValue(1));
  return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1));
}
 
/// Return the packed multiply-halves node for a multiply-parts intrinsic. The
/// scalar spelling maps to the same node; its product is the first element.
static unsigned getRVPMulHalvesOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V multiply-parts intrinsic");
  case Intrinsic::riscv_pmul_00:
  case Intrinsic::riscv_mul_00:
    return RISCVISD::PMUL_HALVES_00;
  case Intrinsic::riscv_pmul_01:
  case Intrinsic::riscv_mul_01:
    return RISCVISD::PMUL_HALVES_01;
  case Intrinsic::riscv_pmul_11:
  case Intrinsic::riscv_mul_11:
    return RISCVISD::PMUL_HALVES_11;
  case Intrinsic::riscv_pmulu_00:
  case Intrinsic::riscv_mulu_00:
    return RISCVISD::PMULU_HALVES_00;
  case Intrinsic::riscv_pmulu_01:
  case Intrinsic::riscv_mulu_01:
    return RISCVISD::PMULU_HALVES_01;
  case Intrinsic::riscv_pmulu_11:
  case Intrinsic::riscv_mulu_11:
    return RISCVISD::PMULU_HALVES_11;
  case Intrinsic::riscv_pmulsu_00:
  case Intrinsic::riscv_mulsu_00:
    return RISCVISD::PMULSU_HALVES_00;
  case Intrinsic::riscv_pmulsu_11:
  case Intrinsic::riscv_mulsu_11:
    return RISCVISD::PMULSU_HALVES_11;
  }
}
 
/// Return the scalar multiply-parts intrinsic computing the first product of
/// packed intrinsic \p IntNo.
static Intrinsic::ID getRVPScalarMulPartsIntrinsic(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V packed multiply-parts intrinsic");
  case Intrinsic::riscv_pmul_00:
    return Intrinsic::riscv_mul_00;
  case Intrinsic::riscv_pmul_01:
    return Intrinsic::riscv_mul_01;
  case Intrinsic::riscv_pmul_11:
    return Intrinsic::riscv_mul_11;
  case Intrinsic::riscv_pmulu_00:
    return Intrinsic::riscv_mulu_00;
  case Intrinsic::riscv_pmulu_01:
    return Intrinsic::riscv_mulu_01;
  case Intrinsic::riscv_pmulu_11:
    return Intrinsic::riscv_mulu_11;
  case Intrinsic::riscv_pmulsu_00:
    return Intrinsic::riscv_mulsu_00;
  case Intrinsic::riscv_pmulsu_11:
    return Intrinsic::riscv_mulsu_11;
  }
}
 
/// Return the accumulate form of multiply-parts intrinsic \p IntNo, or
/// Intrinsic::not_intrinsic if there is none.
static Intrinsic::ID getRVPMulPartsAccIntrinsic(unsigned IntNo) {
  switch (IntNo) {
  default:
    return Intrinsic::not_intrinsic;
  case Intrinsic::riscv_mul_00:
    return Intrinsic::riscv_macc_00;
  case Intrinsic::riscv_pmul_00:
    return Intrinsic::riscv_pmacc_00;
  case Intrinsic::riscv_mul_01:
    return Intrinsic::riscv_macc_01;
  case Intrinsic::riscv_pmul_01:
    return Intrinsic::riscv_pmacc_01;
  case Intrinsic::riscv_mul_11:
    return Intrinsic::riscv_macc_11;
  case Intrinsic::riscv_pmul_11:
    return Intrinsic::riscv_pmacc_11;
  case Intrinsic::riscv_mulu_00:
    return Intrinsic::riscv_maccu_00;
  case Intrinsic::riscv_pmulu_00:
    return Intrinsic::riscv_pmaccu_00;
  case Intrinsic::riscv_mulu_01:
    return Intrinsic::riscv_maccu_01;
  case Intrinsic::riscv_pmulu_01:
    return Intrinsic::riscv_pmaccu_01;
  case Intrinsic::riscv_mulu_11:
    return Intrinsic::riscv_maccu_11;
  case Intrinsic::riscv_pmulu_11:
    return Intrinsic::riscv_pmaccu_11;
  case Intrinsic::riscv_mulsu_00:
    return Intrinsic::riscv_maccsu_00;
  case Intrinsic::riscv_pmulsu_00:
    return Intrinsic::riscv_pmaccsu_00;
  case Intrinsic::riscv_mulsu_11:
    return Intrinsic::riscv_maccsu_11;
  case Intrinsic::riscv_pmulsu_11:
    return Intrinsic::riscv_pmaccsu_11;
  }
}
 
/// Return the multiply-parts accumulate node for \p IntNo.
static unsigned getRVPMulAccHalvesOpcode(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V multiply-parts accumulate intrinsic");
  case Intrinsic::riscv_pmacc_00:
  case Intrinsic::riscv_macc_00:
    return RISCVISD::PMACC_HALVES_00;
  case Intrinsic::riscv_pmacc_01:
  case Intrinsic::riscv_macc_01:
    return RISCVISD::PMACC_HALVES_01;
  case Intrinsic::riscv_pmacc_11:
  case Intrinsic::riscv_macc_11:
    return RISCVISD::PMACC_HALVES_11;
  case Intrinsic::riscv_pmaccu_00:
  case Intrinsic::riscv_maccu_00:
    return RISCVISD::PMACCU_HALVES_00;
  case Intrinsic::riscv_pmaccu_01:
  case Intrinsic::riscv_maccu_01:
    return RISCVISD::PMACCU_HALVES_01;
  case Intrinsic::riscv_pmaccu_11:
  case Intrinsic::riscv_maccu_11:
    return RISCVISD::PMACCU_HALVES_11;
  case Intrinsic::riscv_pmaccsu_00:
  case Intrinsic::riscv_maccsu_00:
    return RISCVISD::PMACCSU_HALVES_00;
  case Intrinsic::riscv_pmaccsu_11:
  case Intrinsic::riscv_maccsu_11:
    return RISCVISD::PMACCSU_HALVES_11;
  }
}
 
/// Return {opcode, rs1 lane, rs2 lane} for the word form of \p IntNo.
static std::tuple<unsigned, unsigned, unsigned>
getRVPWordMulPartsOpcodeAndLanes(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V multiply-parts intrinsic");
  case Intrinsic::riscv_mul_00:
    return {ISD::SMUL_LOHI, 0, 0};
  case Intrinsic::riscv_mul_01:
    return {ISD::SMUL_LOHI, 0, 1};
  case Intrinsic::riscv_mul_11:
    return {ISD::SMUL_LOHI, 1, 1};
  case Intrinsic::riscv_mulu_00:
    return {ISD::UMUL_LOHI, 0, 0};
  case Intrinsic::riscv_mulu_01:
    return {ISD::UMUL_LOHI, 0, 1};
  case Intrinsic::riscv_mulu_11:
    return {ISD::UMUL_LOHI, 1, 1};
  case Intrinsic::riscv_mulsu_00:
    return {RISCVISD::WMULSU, 0, 0};
  case Intrinsic::riscv_mulsu_11:
    return {RISCVISD::WMULSU, 1, 1};
  }
}
 
/// Return {opcode, rs1 lane, rs2 lane} for the word form of accumulate
/// intrinsic \p IntNo.
static std::tuple<unsigned, unsigned, unsigned>
getRVPWordMulPartsAccOpcodeAndLanes(unsigned IntNo) {
  switch (IntNo) {
  default:
    llvm_unreachable("Unexpected RISC-V multiply-parts accumulate intrinsic");
  case Intrinsic::riscv_macc_00:
    return {RISCVISD::WMACC, 0, 0};
  case Intrinsic::riscv_macc_01:
    return {RISCVISD::WMACC, 0, 1};
  case Intrinsic::riscv_macc_11:
    return {RISCVISD::WMACC, 1, 1};
  case Intrinsic::riscv_maccu_00:
    return {RISCVISD::WMACCU, 0, 0};
  case Intrinsic::riscv_maccu_01:
    return {RISCVISD::WMACCU, 0, 1};
  case Intrinsic::riscv_maccu_11:
    return {RISCVISD::WMACCU, 1, 1};
  case Intrinsic::riscv_maccsu_00:
    return {RISCVISD::WMACCSU, 0, 0};
  case Intrinsic::riscv_maccsu_11:
    return {RISCVISD::WMACCSU, 1, 1};
  }
}
 
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                     SelectionDAG &DAG) const {
  unsigned IntNo = Op.getConstantOperandVal(0);
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
 
  switch (IntNo) {
  default:
    break; // Don't custom lower most intrinsics.
  case Intrinsic::riscv_tuple_insert: {
    SDValue Vec = Op.getOperand(1);
    SDValue SubVec = Op.getOperand(2);
    SDValue Index = Op.getOperand(3);
 
    return DAG.getNode(RISCVISD::TUPLE_INSERT, DL, Op.getValueType(), Vec,
                       SubVec, Index);
  }
  case Intrinsic::riscv_tuple_extract: {
    SDValue Vec = Op.getOperand(1);
    SDValue Index = Op.getOperand(2);
 
    return DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, Op.getValueType(), Vec,
                       Index);
  }
  case Intrinsic::thread_pointer: {
    EVT PtrVT = getPointerTy(DAG.getDataLayout());
    return DAG.getRegister(RISCV::X4, PtrVT);
  }
  case Intrinsic::riscv_pmul_00:
  case Intrinsic::riscv_pmul_01:
  case Intrinsic::riscv_pmul_11:
  case Intrinsic::riscv_pmulu_00:
  case Intrinsic::riscv_pmulu_01:
  case Intrinsic::riscv_pmulu_11:
  case Intrinsic::riscv_pmulsu_00:
  case Intrinsic::riscv_pmulsu_11: {
    MVT VT = Op.getSimpleValueType();
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
    unsigned Opc = getRVPMulHalvesOpcode(IntNo);
    if (!Subtarget.isPExtPackedDoubleType(VT))
      return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
 
    // On RV32 a 64-bit result lives in a GPR pair; compute each half with the
    // 32-bit form of the same product.
    auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
    auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
    if (VT == MVT::v2i32) {
      // Each half is a single product, described by the scalar intrinsic.
      SDValue Id = DAG.getTargetConstant(getRVPScalarMulPartsIntrinsic(IntNo),
                                         DL, MVT::i32);
      SDValue Lo =
          DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, Id, Rs1Lo, Rs2Lo);
      SDValue Hi =
          DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, Id, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
    MVT HalfVT = VT.getHalfNumVectorElementsVT();
    SDValue Lo = DAG.getNode(Opc, DL, HalfVT, Rs1Lo, Rs2Lo);
    SDValue Hi = DAG.getNode(Opc, DL, HalfVT, Rs1Hi, Rs2Hi);
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
  }
  case Intrinsic::riscv_pmacc_00:
  case Intrinsic::riscv_pmacc_01:
  case Intrinsic::riscv_pmacc_11:
  case Intrinsic::riscv_pmaccu_00:
  case Intrinsic::riscv_pmaccu_01:
  case Intrinsic::riscv_pmaccu_11:
  case Intrinsic::riscv_pmaccsu_00:
  case Intrinsic::riscv_pmaccsu_11:
  case Intrinsic::riscv_macc_00:
  case Intrinsic::riscv_macc_01:
  case Intrinsic::riscv_macc_11:
  case Intrinsic::riscv_maccu_00:
  case Intrinsic::riscv_maccu_01:
  case Intrinsic::riscv_maccu_11:
  case Intrinsic::riscv_maccsu_00:
  case Intrinsic::riscv_maccsu_11: {
    MVT VT = Op.getSimpleValueType();
    SDValue Rd = Op.getOperand(1);
    SDValue Rs1 = Op.getOperand(2);
    SDValue Rs2 = Op.getOperand(3);
    unsigned Opc = getRVPMulAccHalvesOpcode(IntNo);
    if (VT != MVT::v2i32 || !Subtarget.isPExtPackedDoubleType(VT))
      return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
 
    // On RV32 a 64-bit result lives in a GPR pair; accumulate each half with
    // the 32-bit form of the same product.
    auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
    auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
    SDValue Lo =
        DAG.getNode(Opc, DL, MVT::i32,
                    DAG.getExtractVectorElt(DL, MVT::i32, Rd, 0), Rs1Lo, Rs2Lo);
    SDValue Hi =
        DAG.getNode(Opc, DL, MVT::i32,
                    DAG.getExtractVectorElt(DL, MVT::i32, Rd, 1), Rs1Hi, Rs2Hi);
    return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
  }
  case Intrinsic::riscv_pas:
  case Intrinsic::riscv_psa:
  case Intrinsic::riscv_psas:
  case Intrinsic::riscv_pssa:
  case Intrinsic::riscv_paas:
  case Intrinsic::riscv_pasa: {
    // v2i32 has no paired instruction on RV32; split into a pair of i32 ops
    // with cross-lane operands. The exchange shape is: even result uses
    // (S1[0], S2[1]); odd result uses (S1[1], S2[0]).
    if (Subtarget.is64Bit() || Op.getSimpleValueType() != MVT::v2i32)
      break;
 
    unsigned EvenOpc, OddOpc;
    switch (IntNo) {
    case Intrinsic::riscv_pas:
      EvenOpc = ISD::SUB;
      OddOpc = ISD::ADD;
      break;
    case Intrinsic::riscv_psa:
      EvenOpc = ISD::ADD;
      OddOpc = ISD::SUB;
      break;
    case Intrinsic::riscv_psas:
      EvenOpc = ISD::SSUBSAT;
      OddOpc = ISD::SADDSAT;
      break;
    case Intrinsic::riscv_pssa:
      EvenOpc = ISD::SADDSAT;
      OddOpc = ISD::SSUBSAT;
      break;
    case Intrinsic::riscv_paas:
      EvenOpc = RISCVISD::ASUB;
      OddOpc = ISD::AVGFLOORS;
      break;
    case Intrinsic::riscv_pasa:
      EvenOpc = ISD::AVGFLOORS;
      OddOpc = RISCVISD::ASUB;
      break;
    default:
      llvm_unreachable("Unexpected exchanged add/sub intrinsic");
    }
 
    SDValue S1 = Op.getOperand(1);
    SDValue S2 = Op.getOperand(2);
    SDValue S1Even = DAG.getExtractVectorElt(DL, MVT::i32, S1, 0);
    SDValue S1Odd = DAG.getExtractVectorElt(DL, MVT::i32, S1, 1);
    SDValue S2Even = DAG.getExtractVectorElt(DL, MVT::i32, S2, 0);
    SDValue S2Odd = DAG.getExtractVectorElt(DL, MVT::i32, S2, 1);
 
    SDValue REven = DAG.getNode(EvenOpc, DL, MVT::i32, S1Even, S2Odd);
    SDValue ROdd = DAG.getNode(OddOpc, DL, MVT::i32, S1Odd, S2Even);
    return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, REven, ROdd);
  }
  case Intrinsic::riscv_orc_b:
  case Intrinsic::riscv_brev8:
  case Intrinsic::riscv_sha256sig0:
  case Intrinsic::riscv_sha256sig1:
  case Intrinsic::riscv_sha256sum0:
  case Intrinsic::riscv_sha256sum1:
  case Intrinsic::riscv_sm3p0:
  case Intrinsic::riscv_sm3p1: {
    unsigned Opc;
    switch (IntNo) {
    case Intrinsic::riscv_orc_b:      Opc = RISCVISD::ORC_B;      break;
    case Intrinsic::riscv_brev8:      Opc = RISCVISD::BREV8;      break;
    case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
    case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
    case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
    case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
    case Intrinsic::riscv_sm3p0:      Opc = RISCVISD::SM3P0;      break;
    case Intrinsic::riscv_sm3p1:      Opc = RISCVISD::SM3P1;      break;
    }
 
    return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
  }
  case Intrinsic::riscv_sm4ks:
  case Intrinsic::riscv_sm4ed: {
    unsigned Opc =
        IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
 
    return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2),
                       Op.getOperand(3));
  }
  case Intrinsic::riscv_zip:
  case Intrinsic::riscv_unzip: {
    unsigned Opc =
        IntNo == Intrinsic::riscv_zip ? RISCVISD::ZIP : RISCVISD::UNZIP;
    return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
  }
  case Intrinsic::riscv_mopr:
    return DAG.getNode(RISCVISD::MOP_R, DL, XLenVT, Op.getOperand(1),
                       Op.getOperand(2));
 
  case Intrinsic::riscv_moprr: {
    return DAG.getNode(RISCVISD::MOP_RR, DL, XLenVT, Op.getOperand(1),
                       Op.getOperand(2), Op.getOperand(3));
  }
  case Intrinsic::riscv_clmulh:
  case Intrinsic::riscv_clmulr: {
    unsigned Opc = IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
    return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::riscv_paadd:
  case Intrinsic::riscv_paaddu:
  case Intrinsic::riscv_pasub:
  case Intrinsic::riscv_pasubu:
  case Intrinsic::riscv_pabd:
  case Intrinsic::riscv_pabdu:
  case Intrinsic::riscv_psabs: {
    unsigned Opc;
    switch (IntNo) {
    case Intrinsic::riscv_paadd:
      Opc = ISD::AVGFLOORS;
      break;
    case Intrinsic::riscv_paaddu:
      Opc = ISD::AVGFLOORU;
      break;
    case Intrinsic::riscv_pasub:
      Opc = RISCVISD::ASUB;
      break;
    case Intrinsic::riscv_pasubu:
      Opc = RISCVISD::ASUBU;
      break;
    case Intrinsic::riscv_pabd:
      Opc = ISD::ABDS;
      break;
    case Intrinsic::riscv_pabdu:
      Opc = ISD::ABDU;
      break;
    case Intrinsic::riscv_psabs:
      Opc = RISCVISD::PSABS;
      break;
    }
 
    if (IntNo == Intrinsic::riscv_psabs)
      return DAG.getNode(Opc, DL, Op.getValueType(), Op.getOperand(1));
 
    return DAG.getNode(Opc, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  }
  case Intrinsic::riscv_pnclipp:
  case Intrinsic::riscv_pnclipup: {
    bool IsSigned = IntNo == Intrinsic::riscv_pnclipp;
    EVT VT = Op.getValueType();
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
    if (Subtarget.is64Bit()) {
      if (VT == MVT::v2i32 && !Rs1.getValueType().isVector()) {
        unsigned WOpc = IsSigned ? RISCVISD::PNCLIPP_W : RISCVISD::PNCLIPUP_W;
        return DAG.getNode(WOpc, DL, VT, Rs1, Rs2);
      }
      unsigned Opc = IsSigned ? RISCVISD::PNCLIPP : RISCVISD::PNCLIPUP;
      return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
    }
 
    MVT XLenVT = Subtarget.getXLenVT();
    SDValue Shift = DAG.getTargetConstant(0, DL, XLenVT);
    if (VT == MVT::v4i8) {
      unsigned ClipOpc = IsSigned ? RISCVISD::PNCLIP : RISCVISD::PNCLIPU;
      SDValue Pair = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, Rs1, Rs2);
      return DAG.getNode(ClipOpc, DL, VT, Pair, Shift);
    }
    if (VT == MVT::v2i16) {
      unsigned ClipOpc = IsSigned ? RISCVISD::PNCLIP : RISCVISD::PNCLIPU;
      SDValue Pair = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, Rs1, Rs2);
      return DAG.getNode(ClipOpc, DL, VT, Pair, Shift);
    }
    if (VT == MVT::v2i32) {
      unsigned ClipOpc = IsSigned ? RISCVISD::NCLIP : RISCVISD::NCLIPU;
      auto [Rs1Lo, Rs1Hi] = DAG.SplitScalar(Rs1, DL, XLenVT, XLenVT);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitScalar(Rs2, DL, XLenVT, XLenVT);
      SDValue Lo = DAG.getNode(ClipOpc, DL, XLenVT, Rs1Lo, Rs1Hi, Shift);
      SDValue Hi = DAG.getNode(ClipOpc, DL, XLenVT, Rs2Lo, Rs2Hi, Shift);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
    if (VT == MVT::v8i8 || VT == MVT::v4i16) {
      unsigned ClipOpc = IsSigned ? RISCVISD::PNCLIP : RISCVISD::PNCLIPU;
      MVT HalfVT = VT == MVT::v8i8 ? MVT::v4i8 : MVT::v2i16;
      SDValue Lo = DAG.getNode(ClipOpc, DL, HalfVT, Rs1, Shift);
      SDValue Hi = DAG.getNode(ClipOpc, DL, HalfVT, Rs2, Shift);
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
 
    llvm_unreachable("unexpected VT for pnclipp/pnclipup on RV32");
  }
  case Intrinsic::riscv_mqacc_00:
  case Intrinsic::riscv_mqacc_01:
  case Intrinsic::riscv_mqacc_11:
  case Intrinsic::riscv_mqracc_00:
  case Intrinsic::riscv_mqracc_01:
  case Intrinsic::riscv_mqracc_11:
  case Intrinsic::riscv_pmqacc_h00:
  case Intrinsic::riscv_pmqacc_h01:
  case Intrinsic::riscv_pmqacc_h11:
  case Intrinsic::riscv_pmqracc_h00:
  case Intrinsic::riscv_pmqracc_h01:
  case Intrinsic::riscv_pmqracc_h11: {
    EVT VT = Op.getValueType();
    SDValue Rd = Op.getOperand(1);
    SDValue Rs1 = Op.getOperand(2);
    SDValue Rs2 = Op.getOperand(3);
    MVT XLenVT = Subtarget.getXLenVT();
 
    if (VT == MVT::v2i32 && Rs1.getSimpleValueType() == MVT::v4i16) {
      if (Subtarget.is64Bit()) {
        unsigned Opc = getRVPQFormatAccOpcode(IntNo);
        return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
      }
 
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      SDValue RdLo = DAG.getExtractVectorElt(DL, XLenVT, Rd, 0);
      SDValue RdHi = DAG.getExtractVectorElt(DL, XLenVT, Rd, 1);
      unsigned ScalarOpc = getRVPQFormatAccScalarOpcode(IntNo);
      SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo, Rs1Lo, Rs2Lo);
      SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    if (VT == XLenVT) {
      unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo);
      return DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2);
    }
 
    return SDValue();
  }
  case Intrinsic::riscv_pmulq:
  case Intrinsic::riscv_pmulqr: {
    unsigned Opc;
    switch (IntNo) {
    case Intrinsic::riscv_pmulq:
      Opc = RISCVISD::MULQ;
      break;
    case Intrinsic::riscv_pmulqr:
      Opc = RISCVISD::MULQR;
      break;
    }
 
    EVT VT = Op.getValueType();
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
 
    // On RV32 the 64-bit packed vectors (v2i32, v4i16) have no single Q-format
    // multiply instruction. Split v4i16 into two v2i16 halves (each lowered to
    // a single pmulq.h/pmulqr.h via the existing patterns), and v2i32 into two
    // scalar mulq/mulqr (matched by the RV32 PatGprGpr patterns).
    if (!Subtarget.is64Bit()) {
      if (VT == MVT::v2i32) {
        MVT XLenVT = Subtarget.getXLenVT();
        SDValue Lo1 = DAG.getExtractVectorElt(DL, XLenVT, Rs1, 0);
        SDValue Lo2 = DAG.getExtractVectorElt(DL, XLenVT, Rs2, 0);
        SDValue Hi1 = DAG.getExtractVectorElt(DL, XLenVT, Rs1, 1);
        SDValue Hi2 = DAG.getExtractVectorElt(DL, XLenVT, Rs2, 1);
        SDValue LoRes = DAG.getNode(Opc, DL, XLenVT, Lo1, Lo2);
        SDValue HiRes = DAG.getNode(Opc, DL, XLenVT, Hi1, Hi2);
        return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, LoRes, HiRes);
      }
      if (VT == MVT::v4i16) {
        auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
        auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
        SDValue LoRes = DAG.getNode(Opc, DL, MVT::v2i16, Rs1Lo, Rs2Lo);
        SDValue HiRes = DAG.getNode(Opc, DL, MVT::v2i16, Rs1Hi, Rs2Hi);
        return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoRes, HiRes);
      }
    }
 
    return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
  }
  case Intrinsic::riscv_psll:
  case Intrinsic::riscv_psrl:
  case Intrinsic::riscv_psra:
  case Intrinsic::riscv_pssha:
  case Intrinsic::riscv_psshar:
  case Intrinsic::riscv_psshl:
  case Intrinsic::riscv_psshlr: {
    SDValue ShAmt = Op.getOperand(2);
    ShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, XLenVT);
    return DAG.getNode(getRVPShiftOpcode(IntNo), DL, Op.getValueType(),
                       Op.getOperand(1), ShAmt);
  }
  case Intrinsic::riscv_pwsll:
  case Intrinsic::riscv_pwsla: {
    MVT VT = Op.getSimpleValueType();
    SDValue Src = Op.getOperand(1);
    MVT SrcVT = Src.getSimpleValueType();
    if (!((VT == MVT::v4i16 && SrcVT == MVT::v4i8) ||
          (VT == MVT::v2i32 && SrcVT == MVT::v2i16)))
      reportFatalUsageError("unsupported packed widening shift intrinsic");
 
    SDValue ShAmt = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op.getOperand(2));
    bool IsSigned = IntNo == Intrinsic::riscv_pwsla;
    if (!Subtarget.is64Bit()) {
      unsigned Opc = IsSigned ? RISCVISD::PWSLA : RISCVISD::PWSLL;
      return DAG.getNode(Opc, DL, VT, Src, ShAmt);
    }
 
    unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    SDValue Wide = DAG.getNode(ExtOpc, DL, VT, Src);
    return DAG.getNode(RISCVISD::PSLL, DL, VT, Wide, ShAmt);
  }
  case Intrinsic::riscv_pnsrl:
  case Intrinsic::riscv_pnsra:
  case Intrinsic::riscv_pnsrar: {
    MVT VT = Op.getSimpleValueType();
    MVT SrcVT = Op.getOperand(1).getSimpleValueType();
    if (!((VT == MVT::v4i8 && SrcVT == MVT::v4i16) ||
          (VT == MVT::v2i16 && SrcVT == MVT::v2i32)))
      reportFatalUsageError("unsupported packed narrowing shift intrinsic");
 
    SDValue ShAmt = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op.getOperand(2));
    return DAG.getNode(getRVPNarrowingShiftOpcode(IntNo), DL, VT,
                       Op.getOperand(1), ShAmt);
  }
  case Intrinsic::riscv_psati:
  case Intrinsic::riscv_pusati: {
    bool IsSigned = IntNo == Intrinsic::riscv_psati;
    unsigned Opc = IsSigned ? RISCVISD::SATI : RISCVISD::USATI;
    // psati's width counts the sign bit, RISCVISD::SATI's immediate does not.
    unsigned Width = Op.getConstantOperandVal(2) - (IsSigned ? 1 : 0);
    return DAG.getNode(Opc, DL, Op.getValueType(), Op.getOperand(1),
                       DAG.getTargetConstant(Width, DL, XLenVT));
  }
  case Intrinsic::riscv_psext_b:
  case Intrinsic::riscv_psext_h: {
    EVT VT = Op.getValueType();
    if (!VT.isSimple() || !Subtarget.isPExtPackedType(VT.getSimpleVT()))
      reportFatalUsageError("unsupported llvm.riscv.psext intrinsic");
 
    MVT SimpleVT = VT.getSimpleVT();
    unsigned SrcEltBits = IntNo == Intrinsic::riscv_psext_b ? 8 : 16;
    if (SrcEltBits >= SimpleVT.getScalarSizeInBits())
      reportFatalUsageError("unsupported llvm.riscv.psext intrinsic");
 
    MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltBits),
                                 SimpleVT.getVectorElementCount());
    return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SimpleVT, Op.getOperand(1),
                       DAG.getValueType(ExtVT));
  }
  case Intrinsic::riscv_pzext_b:
  case Intrinsic::riscv_pzext_h: {
    EVT VT = Op.getValueType();
    if (!VT.isSimple() || !Subtarget.isPExtPackedType(VT.getSimpleVT()))
      reportFatalUsageError("unsupported llvm.riscv.pzext intrinsic");
 
    MVT SimpleVT = VT.getSimpleVT();
    unsigned SrcEltBits = IntNo == Intrinsic::riscv_pzext_b ? 8 : 16;
    if (SimpleVT.getScalarSizeInBits() != SrcEltBits * 2)
      reportFatalUsageError("unsupported llvm.riscv.pzext intrinsic");
 
    return lowerPZExt(Op.getOperand(1), DL, DAG, Subtarget);
  }
  case Intrinsic::riscv_pabdsumu:
  case Intrinsic::riscv_pabdsumau: {
    // On RV32 an i32-result absolute difference sum over a 64-bit (GPRPair)
    // source has no paired instruction. Split into two v4i8 halves: reduce the
    // low half (folding in rd when accumulating), then accumulate the high half
    // into that partial sum.
    SDValue Rs1 = Op.getOperand(Op.getNumOperands() - 2);
    SDValue Rs2 = Op.getOperand(Op.getNumOperands() - 1);
    if (Subtarget.is64Bit() || Rs1.getSimpleValueType() != MVT::v8i8)
      return SDValue();
    bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
    auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
    auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
    SDValue AbdsumuId =
        DAG.getTargetConstant(Intrinsic::riscv_pabdsumu, DL, MVT::i32);
    SDValue AbdsumauId =
        DAG.getTargetConstant(Intrinsic::riscv_pabdsumau, DL, MVT::i32);
    SDValue Lo = IsAcc ? DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                                     AbdsumauId, Op.getOperand(1), Rs1Lo, Rs2Lo)
                       : DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
                                     AbdsumuId, Rs1Lo, Rs2Lo);
    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, AbdsumauId, Lo,
                       Rs1Hi, Rs2Hi);
  }
  case Intrinsic::riscv_pmulhr:
  case Intrinsic::riscv_pmulhru:
  case Intrinsic::riscv_pmulhsu:
  case Intrinsic::riscv_pmulhrsu: {
    EVT VT = Op.getValueType();
    unsigned Opc = getRVPMulHighOpcode(IntNo);
 
    // RV32 has no single instruction for 64-bit packed multiply high. Split
    // v4i16 into two v2i16 packed operations, and split v2i32 into scalar i32
    // operations so isel can use MULH*.
    if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Op.getOperand(1), DL);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Op.getOperand(2), DL);
      SDValue Lo = DAG.getNode(Opc, DL, MVT::v2i16, Rs1Lo, Rs2Lo);
      SDValue Hi = DAG.getNode(Opc, DL, MVT::v2i16, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
 
    if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
      auto Extract = [&](SDValue V, unsigned Idx) {
        return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
      };
      SDValue Rs1 = Op.getOperand(1);
      SDValue Rs2 = Op.getOperand(2);
      SDValue Lo =
          DAG.getNode(Opc, DL, MVT::i32, Extract(Rs1, 0), Extract(Rs2, 0));
      SDValue Hi =
          DAG.getNode(Opc, DL, MVT::i32, Extract(Rs1, 1), Extract(Rs2, 1));
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    return DAG.getNode(Opc, DL, VT, Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::riscv_mulhr_i32:
  case Intrinsic::riscv_mulhru_u32:
  case Intrinsic::riscv_mulhsu_i32:
  case Intrinsic::riscv_mulhrsu_i32: {
    // RV32 maps the non-rounding forms onto the M extension and the rounding
    // forms onto the scalar P instructions. RV64 goes via ReplaceNodeResults.
    unsigned Opc = getRVScalarMulHighOpcode(IntNo);
    return DAG.getNode(Opc, DL, MVT::i32, Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::riscv_pmhacc:
  case Intrinsic::riscv_pmhracc:
  case Intrinsic::riscv_pmhaccu:
  case Intrinsic::riscv_pmhraccu:
  case Intrinsic::riscv_pmhaccsu:
  case Intrinsic::riscv_pmhraccsu: {
    EVT VT = Op.getValueType();
    unsigned MulOpc = getRVPMulHighAccumulateOpcode(IntNo);
    SDValue Rd = Op.getOperand(1);
    SDValue Rs1 = Op.getOperand(2);
    SDValue Rs2 = Op.getOperand(3);
 
    // RV32 has no single instruction for 64-bit packed multiply high
    // accumulate. Split v4i16 into two v2i16 packed operations, and split
    // v2i32 into scalar i32 operations.
    if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
      auto [RdLo, RdHi] = DAG.SplitVector(Rd, DL);
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      SDValue Lo = DAG.getNode(MulOpc, DL, MVT::v2i16, RdLo, Rs1Lo, Rs2Lo);
      SDValue Hi = DAG.getNode(MulOpc, DL, MVT::v2i16, RdHi, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
 
    if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
      auto Extract = [&](SDValue V, unsigned Idx) {
        return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
      };
      SDValue Lo = DAG.getNode(MulOpc, DL, MVT::i32, Extract(Rd, 0),
                               Extract(Rs1, 0), Extract(Rs2, 0));
      SDValue Hi = DAG.getNode(MulOpc, DL, MVT::i32, Extract(Rd, 1),
                               Extract(Rs1, 1), Extract(Rs2, 1));
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    return DAG.getNode(MulOpc, DL, VT, Rd, Rs1, Rs2);
  }
  case Intrinsic::riscv_pmhacc_b0:
  case Intrinsic::riscv_pmhacc_b1:
  case Intrinsic::riscv_pmhaccsu_b0:
  case Intrinsic::riscv_pmhaccsu_b1: {
    EVT VT = Op.getValueType();
    unsigned Opc = getRVPMulHighAccumulateByHalvesOpcode(IntNo);
    SDValue Rd = Op.getOperand(1);
    SDValue Rs1 = Op.getOperand(2);
    SDValue Rs2 = Op.getOperand(3);
 
    // RV32: split v4i16 into two v2i16 operations
    if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
      auto [RdLo, RdHi] = DAG.SplitVector(Rd, DL);
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      SDValue Lo = DAG.getNode(Opc, DL, MVT::v2i16, RdLo, Rs1Lo, Rs2Lo);
      SDValue Hi = DAG.getNode(Opc, DL, MVT::v2i16, RdHi, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
 
    return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
  }
  case Intrinsic::riscv_pmhacc_h0:
  case Intrinsic::riscv_pmhacc_h1:
  case Intrinsic::riscv_pmhaccsu_h0:
  case Intrinsic::riscv_pmhaccsu_h1: {
    EVT VT = Op.getValueType();
    unsigned Opc = getRVPMulHighAccumulateByHalvesOpcode(IntNo);
    SDValue Rd = Op.getOperand(1);
    SDValue Rs1 = Op.getOperand(2);
    SDValue Rs2 = Op.getOperand(3);
 
    // RV32 has no 64-bit packed form: split into two scalar operations, each
    // accumulating one word of the result.
    if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
      auto Extract = [&](SDValue V, unsigned Idx) {
        return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
      };
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, Extract(Rd, 0),
                               Extract(Rs1, 0), Rs2Lo);
      SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, Extract(Rd, 1),
                               Extract(Rs1, 1), Rs2Hi);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
  }
  case Intrinsic::riscv_pmulh_b0:
  case Intrinsic::riscv_pmulh_b1:
  case Intrinsic::riscv_pmulhsu_b0:
  case Intrinsic::riscv_pmulhsu_b1: {
    EVT VT = Op.getValueType();
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
    unsigned Opc = getRVPMulHighPartsOpcode(IntNo);
 
    // RV32 has no single instruction for a 64-bit packed multiply-high parts.
    // Split v4i16 into two v2i16 packed operations.
    if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      SDValue Lo = DAG.getNode(Opc, DL, MVT::v2i16, Rs1Lo, Rs2Lo);
      SDValue Hi = DAG.getNode(Opc, DL, MVT::v2i16, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
    }
 
    return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
  }
  case Intrinsic::riscv_pmulh_h0:
  case Intrinsic::riscv_pmulh_h1:
  case Intrinsic::riscv_pmulhsu_h0:
  case Intrinsic::riscv_pmulhsu_h1: {
    EVT VT = Op.getValueType();
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
    unsigned Opc = getRVPMulHighPartsWOpcode(IntNo);
 
    // RV32 has no single instruction for a 64-bit packed word multiply-high
    // parts. Split v2i32 into two scalar operations, one per result word.
    if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
      auto Extract = [&](SDValue V, unsigned Idx) {
        return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
      };
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      unsigned SOpc = getRVPScalarMulHighPartsOpcode(IntNo);
      SDValue Lo = DAG.getNode(SOpc, DL, MVT::i32, Extract(Rs1, 0), Rs2Lo);
      SDValue Hi = DAG.getNode(SOpc, DL, MVT::i32, Extract(Rs1, 1), Rs2Hi);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    if (Subtarget.is64Bit())
      return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
 
    return SDValue();
  }
  case Intrinsic::riscv_pmqwacc_i32x2:
  case Intrinsic::riscv_pmqrwacc_i32x2: {
    EVT VT = Op.getValueType();
    SDValue Acc = Op.getOperand(1);
    SDValue Rs1 = Op.getOperand(2);
    SDValue Rs2 = Op.getOperand(3);
 
    if (!Subtarget.is64Bit()) {
      unsigned Opc = IntNo == Intrinsic::riscv_pmqwacc_i32x2
                         ? RISCVISD::PMQWACC_H
                         : RISCVISD::PMQRWACC_H;
      return DAG.getNode(Opc, DL, VT, Acc, Rs1, Rs2);
    }
 
    Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, Rs1,
                      DAG.getUNDEF(MVT::v2i16));
    Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, Rs2,
                      DAG.getUNDEF(MVT::v2i16));
    SDValue Zip = DAG.getNode(RISCVISD::PZIP, DL, MVT::v4i16, Rs1, Rs2);
    unsigned Opc = IntNo == Intrinsic::riscv_pmqwacc_i32x2
                       ? RISCVISD::PMQACC_W_H01
                       : RISCVISD::PMQRACC_W_H01;
    return DAG.getNode(Opc, DL, VT, Acc, Zip, Zip);
  }
  case Intrinsic::riscv_mulh_h0:
  case Intrinsic::riscv_mulh_h1:
  case Intrinsic::riscv_mulhsu_h0:
  case Intrinsic::riscv_mulhsu_h1: {
    // mulh.h0/h1 exist only on RV32 and pmulh.w.h0/h1 only on RV64; the
    // other XLEN lowers the scalar intrinsic via the packed form in
    // ReplaceNodeResults.
    EVT VT = Op.getValueType();
    if (VT != MVT::i32 || Subtarget.is64Bit())
      return SDValue();
 
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
    unsigned Opc = getRVPScalarMulHighPartsOpcode(IntNo);
    return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
  }
  case Intrinsic::riscv_pm4add:
  case Intrinsic::riscv_pm2add:
  case Intrinsic::riscv_pm2add_x:
  case Intrinsic::riscv_pm4addu:
  case Intrinsic::riscv_pm2addu:
  case Intrinsic::riscv_pmq2add:
  case Intrinsic::riscv_pmqr2add:
  case Intrinsic::riscv_pm2sadd:
  case Intrinsic::riscv_pm2sadd_x:
  case Intrinsic::riscv_pm2sub:
  case Intrinsic::riscv_pm2sub_x:
  case Intrinsic::riscv_pm4addsu:
  case Intrinsic::riscv_pm2addsu: {
    EVT VT = Op.getValueType();
    unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
    SDValue Rs1 = Op.getOperand(1);
    SDValue Rs2 = Op.getOperand(2);
 
    // RV32 applies the 32-bit instruction independently to both halves of a
    // 64-bit packed input.
    if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
      SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, Rs1Lo, Rs2Lo);
      SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, Rs1Hi, Rs2Hi);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
  }
  case Intrinsic::riscv_pmerge: {
    EVT VT = Op.getValueType();
    auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
                          EVT ResultVT) {
      MVT IntVT = MVT::getIntegerVT(ResultVT.getSizeInBits());
      SDValue Res =
          DAG.getNode(RISCVISD::MERGE, DL, IntVT, DAG.getBitcast(IntVT, Mask),
                      DAG.getBitcast(IntVT, Rs1), DAG.getBitcast(IntVT, Rs2));
      return DAG.getBitcast(ResultVT, Res);
    };
 
    // 64-bit packed types on RV32: split into two 32-bit halves. v2i32 has no
    // legal 32-bit vector half, so bitcast it to v4i16 (same 64 bits) first;
    // the merge result is identical.
    if (!Subtarget.is64Bit() &&
        (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32)) {
      EVT WorkVT = VT == MVT::v2i32 ? EVT(MVT::v4i16) : VT;
      SDValue Rs1 = DAG.getBitcast(WorkVT, Op.getOperand(1));
      SDValue Rs2 = DAG.getBitcast(WorkVT, Op.getOperand(2));
      SDValue Mask = DAG.getBitcast(WorkVT, Op.getOperand(3));
      MVT HalfVT = WorkVT == MVT::v8i8 ? MVT::v4i8 : MVT::v2i16;
      auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL, HalfVT, HalfVT);
      auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL, HalfVT, HalfVT);
      auto [MaskLo, MaskHi] = DAG.SplitVector(Mask, DL, HalfVT, HalfVT);
      SDValue ResLo = buildMerge(Rs1Lo, Rs2Lo, MaskLo, HalfVT);
      SDValue ResHi = buildMerge(Rs1Hi, Rs2Hi, MaskHi, HalfVT);
      SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, WorkVT, ResLo, ResHi);
      return DAG.getBitcast(VT, Res);
    }
 
    return buildMerge(Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), VT);
  }
  case Intrinsic::experimental_get_vector_length:
    return lowerGetVectorLength(Op.getNode(), DAG, Subtarget);
  case Intrinsic::riscv_vmv_x_s: {
    SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
    return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
  }
  case Intrinsic::riscv_vfmv_f_s:
    return DAG.getExtractVectorElt(DL, Op.getValueType(), Op.getOperand(1), 0);
  case Intrinsic::riscv_vmv_v_x:
    return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
                            Op.getOperand(3), Op.getSimpleValueType(), DL, DAG,
                            Subtarget);
  case Intrinsic::riscv_vfmv_v_f:
    return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::riscv_vmv_s_x: {
    SDValue Scalar = Op.getOperand(2);
 
    if (Scalar.getValueType().bitsLE(XLenVT)) {
      Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
      return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
                         Op.getOperand(1), Scalar, Op.getOperand(3));
    }
 
    assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
 
    // This is an i64 value that lives in two scalar registers. We have to
    // insert this in a convoluted way. First we build vXi64 splat containing
    // the two values that we assemble using some bit math. Next we'll use
    // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
    // to merge element 0 from our splat into the source vector.
    // FIXME: This is probably not the best way to do this, but it is
    // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
    // point.
    //   sw lo, (a0)
    //   sw hi, 4(a0)
    //   vlse vX, (a0)
    //
    //   vid.v      vVid
    //   vmseq.vx   mMask, vVid, 0
    //   vmerge.vvm vDest, vSrc, vVal, mMask
    MVT VT = Op.getSimpleValueType();
    SDValue Vec = Op.getOperand(1);
    SDValue VL = getVLOperand(Op);
 
    SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG);
    if (Op.getOperand(1).isUndef())
      return SplattedVal;
    SDValue SplattedIdx =
        DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
                    DAG.getConstant(0, DL, MVT::i32), VL);
 
    MVT MaskVT = getMaskTypeFor(VT);
    SDValue Mask = getAllOnesMask(VT, VL, DL, DAG);
    SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
    SDValue SelectCond =
        DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
                    {VID, SplattedIdx, DAG.getCondCode(ISD::SETEQ),
                     DAG.getUNDEF(MaskVT), Mask, VL});
    return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal,
                       Vec, DAG.getUNDEF(VT), VL);
  }
  case Intrinsic::riscv_vfmv_s_f:
    return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  // EGS * EEW >= 128 bits
  case Intrinsic::riscv_vaesdf_vv:
  case Intrinsic::riscv_vaesdf_vs:
  case Intrinsic::riscv_vaesdm_vv:
  case Intrinsic::riscv_vaesdm_vs:
  case Intrinsic::riscv_vaesef_vv:
  case Intrinsic::riscv_vaesef_vs:
  case Intrinsic::riscv_vaesem_vv:
  case Intrinsic::riscv_vaesem_vs:
  case Intrinsic::riscv_vaeskf1:
  case Intrinsic::riscv_vaeskf2:
  case Intrinsic::riscv_vaesz_vs:
  case Intrinsic::riscv_vsm4k:
  case Intrinsic::riscv_vsm4r_vv:
  case Intrinsic::riscv_vsm4r_vs: {
    if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) ||
        !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
        !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
      reportFatalUsageError("EGW should be greater than or equal to 4 * SEW.");
    return Op;
  }
  // EGS * EEW >= 256 bits
  case Intrinsic::riscv_vsm3c:
  case Intrinsic::riscv_vsm3me: {
    if (!isValidEGW(8, Op.getSimpleValueType(), Subtarget) ||
        !isValidEGW(8, Op->getOperand(1).getSimpleValueType(), Subtarget))
      reportFatalUsageError("EGW should be greater than or equal to 8 * SEW.");
    return Op;
  }
  // zvknha(SEW=32)/zvknhb(SEW=[32|64])
  case Intrinsic::riscv_vsha2ch:
  case Intrinsic::riscv_vsha2cl:
  case Intrinsic::riscv_vsha2ms: {
    if (Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
        !Subtarget.hasStdExtZvknhb())
      reportFatalUsageError("SEW=64 needs Zvknhb to be enabled.");
    if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) ||
        !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
        !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
      reportFatalUsageError("EGW should be greater than or equal to 4 * SEW.");
    return Op;
  }
  case Intrinsic::riscv_sf_vc_v_x:
  case Intrinsic::riscv_sf_vc_v_i:
  case Intrinsic::riscv_sf_vc_v_xv:
  case Intrinsic::riscv_sf_vc_v_iv:
  case Intrinsic::riscv_sf_vc_v_vv:
  case Intrinsic::riscv_sf_vc_v_fv:
  case Intrinsic::riscv_sf_vc_v_xvv:
  case Intrinsic::riscv_sf_vc_v_ivv:
  case Intrinsic::riscv_sf_vc_v_vvv:
  case Intrinsic::riscv_sf_vc_v_fvv:
  case Intrinsic::riscv_sf_vc_v_xvw:
  case Intrinsic::riscv_sf_vc_v_ivw:
  case Intrinsic::riscv_sf_vc_v_vvw:
  case Intrinsic::riscv_sf_vc_v_fvw: {
    MVT VT = Op.getSimpleValueType();
 
    SmallVector<SDValue> Operands{Op->op_values()};
    processVCIXOperands(Op, Operands, DAG);
 
    MVT RetVT = VT;
    if (VT.isFixedLengthVector())
      RetVT = getContainerForFixedLengthVector(VT);
    else if (VT.isFloatingPoint())
      RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()),
                               VT.getVectorElementCount());
 
    SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Operands);
 
    if (VT.isFixedLengthVector())
      NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget);
    else if (VT.isFloatingPoint())
      NewNode = DAG.getBitcast(VT, NewNode);
 
    if (Op == NewNode)
      break;
 
    return NewNode;
  }
  }
 
  return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
}
 
static inline SDValue getVCIXISDNodeWCHAIN(SDValue Op, SelectionDAG &DAG,
                                           unsigned Type) {
  SDLoc DL(Op);
  SmallVector<SDValue> Operands{Op->op_values()};
  Operands.erase(Operands.begin() + 1);
 
  const RISCVSubtarget &Subtarget =
      DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
  MVT VT = Op.getSimpleValueType();
  MVT RetVT = VT;
  MVT FloatVT = VT;
 
  if (VT.isFloatingPoint()) {
    RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()),
                             VT.getVectorElementCount());
    FloatVT = RetVT;
  }
  if (VT.isFixedLengthVector())
    RetVT = getContainerForFixedLengthVector(RetVT, Subtarget);
 
  processVCIXOperands(Op, Operands, DAG);
 
  SDVTList VTs = DAG.getVTList({RetVT, MVT::Other});
  SDValue NewNode = DAG.getNode(Type, DL, VTs, Operands);
  SDValue Chain = NewNode.getValue(1);
 
  if (VT.isFixedLengthVector())
    NewNode = convertFromScalableVector(FloatVT, NewNode, DAG, Subtarget);
  if (VT.isFloatingPoint())
    NewNode = DAG.getBitcast(VT, NewNode);
 
  NewNode = DAG.getMergeValues({NewNode, Chain}, DL);
 
  return NewNode;
}
 
static inline SDValue getVCIXISDNodeVOID(SDValue Op, SelectionDAG &DAG,
                                         unsigned Type) {
  SmallVector<SDValue> Operands{Op->op_values()};
  Operands.erase(Operands.begin() + 1);
  processVCIXOperands(Op, Operands, DAG);
 
  return DAG.getNode(Type, SDLoc(Op), Op.getValueType(), Operands);
}
 
static SDValue
lowerFixedVectorSegLoadIntrinsics(unsigned IntNo, SDValue Op,
                                  const RISCVSubtarget &Subtarget,
                                  SelectionDAG &DAG) {
  bool IsStrided;
  switch (IntNo) {
  case Intrinsic::riscv_seg2_load_mask:
  case Intrinsic::riscv_seg3_load_mask:
  case Intrinsic::riscv_seg4_load_mask:
  case Intrinsic::riscv_seg5_load_mask:
  case Intrinsic::riscv_seg6_load_mask:
  case Intrinsic::riscv_seg7_load_mask:
  case Intrinsic::riscv_seg8_load_mask:
    IsStrided = false;
    break;
  case Intrinsic::riscv_sseg2_load_mask:
  case Intrinsic::riscv_sseg3_load_mask:
  case Intrinsic::riscv_sseg4_load_mask:
  case Intrinsic::riscv_sseg5_load_mask:
  case Intrinsic::riscv_sseg6_load_mask:
  case Intrinsic::riscv_sseg7_load_mask:
  case Intrinsic::riscv_sseg8_load_mask:
    IsStrided = true;
    break;
  default:
    llvm_unreachable("unexpected intrinsic ID");
  };
 
  static const Intrinsic::ID VlsegInts[7] = {
      Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
      Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
      Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
      Intrinsic::riscv_vlseg8_mask};
  static const Intrinsic::ID VlssegInts[7] = {
      Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
      Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
      Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
      Intrinsic::riscv_vlsseg8_mask};
 
  SDLoc DL(Op);
  unsigned NF = Op->getNumValues() - 1;
  assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op->getSimpleValueType(0);
  MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
  unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
                ContainerVT.getScalarSizeInBits();
  EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
 
  // Operands: (chain, int_id, pointer, mask, vl) or
  // (chain, int_id, pointer, offset, mask, vl)
  SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
  SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
  MVT MaskVT = Mask.getSimpleValueType();
  MVT MaskContainerVT = ::getContainerForFixedLengthVector(MaskVT, Subtarget);
  Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
 
  SDValue IntID = DAG.getTargetConstant(
      IsStrided ? VlssegInts[NF - 2] : VlsegInts[NF - 2], DL, XLenVT);
  auto *Load = cast<MemIntrinsicSDNode>(Op);
 
  SDVTList VTs = DAG.getVTList({VecTupTy, MVT::Other});
  SmallVector<SDValue, 9> Ops = {
      Load->getChain(),
      IntID,
      DAG.getUNDEF(VecTupTy),
      Op.getOperand(2),
      Mask,
      VL,
      DAG.getTargetConstant(
          RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, XLenVT),
      DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
  // Insert the stride operand.
  if (IsStrided)
    Ops.insert(std::next(Ops.begin(), 4), Op.getOperand(3));
 
  SDValue Result =
      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
                              Load->getMemoryVT(), Load->getMemOperand());
  SmallVector<SDValue, 9> Results;
  for (unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
    SDValue SubVec = DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, ContainerVT,
                                 Result.getValue(0),
                                 DAG.getTargetConstant(RetIdx, DL, MVT::i32));
    Results.push_back(convertFromScalableVector(VT, SubVec, DAG, Subtarget));
  }
  Results.push_back(Result.getValue(1));
  return DAG.getMergeValues(Results, DL);
}
 
SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
                                                    SelectionDAG &DAG) const {
  unsigned IntNo = Op.getConstantOperandVal(1);
  switch (IntNo) {
  default:
    break;
  case Intrinsic::riscv_seg2_load_mask:
  case Intrinsic::riscv_seg3_load_mask:
  case Intrinsic::riscv_seg4_load_mask:
  case Intrinsic::riscv_seg5_load_mask:
  case Intrinsic::riscv_seg6_load_mask:
  case Intrinsic::riscv_seg7_load_mask:
  case Intrinsic::riscv_seg8_load_mask:
  case Intrinsic::riscv_sseg2_load_mask:
  case Intrinsic::riscv_sseg3_load_mask:
  case Intrinsic::riscv_sseg4_load_mask:
  case Intrinsic::riscv_sseg5_load_mask:
  case Intrinsic::riscv_sseg6_load_mask:
  case Intrinsic::riscv_sseg7_load_mask:
  case Intrinsic::riscv_sseg8_load_mask:
    return lowerFixedVectorSegLoadIntrinsics(IntNo, Op, Subtarget, DAG);
 
  case Intrinsic::riscv_sf_vc_v_x_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_X_SE);
  case Intrinsic::riscv_sf_vc_v_i_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_I_SE);
  case Intrinsic::riscv_sf_vc_v_xv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XV_SE);
  case Intrinsic::riscv_sf_vc_v_iv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IV_SE);
  case Intrinsic::riscv_sf_vc_v_vv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VV_SE);
  case Intrinsic::riscv_sf_vc_v_fv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FV_SE);
  case Intrinsic::riscv_sf_vc_v_xvv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XVV_SE);
  case Intrinsic::riscv_sf_vc_v_ivv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IVV_SE);
  case Intrinsic::riscv_sf_vc_v_vvv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VVV_SE);
  case Intrinsic::riscv_sf_vc_v_fvv_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FVV_SE);
  case Intrinsic::riscv_sf_vc_v_xvw_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XVW_SE);
  case Intrinsic::riscv_sf_vc_v_ivw_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IVW_SE);
  case Intrinsic::riscv_sf_vc_v_vvw_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VVW_SE);
  case Intrinsic::riscv_sf_vc_v_fvw_se:
    return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FVW_SE);
  }
 
  return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
}
 
static SDValue
lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
                                   const RISCVSubtarget &Subtarget,
                                   SelectionDAG &DAG) {
  bool IsStrided;
  switch (IntNo) {
  case Intrinsic::riscv_seg2_store_mask:
  case Intrinsic::riscv_seg3_store_mask:
  case Intrinsic::riscv_seg4_store_mask:
  case Intrinsic::riscv_seg5_store_mask:
  case Intrinsic::riscv_seg6_store_mask:
  case Intrinsic::riscv_seg7_store_mask:
  case Intrinsic::riscv_seg8_store_mask:
    IsStrided = false;
    break;
  case Intrinsic::riscv_sseg2_store_mask:
  case Intrinsic::riscv_sseg3_store_mask:
  case Intrinsic::riscv_sseg4_store_mask:
  case Intrinsic::riscv_sseg5_store_mask:
  case Intrinsic::riscv_sseg6_store_mask:
  case Intrinsic::riscv_sseg7_store_mask:
  case Intrinsic::riscv_sseg8_store_mask:
    IsStrided = true;
    break;
  default:
    llvm_unreachable("unexpected intrinsic ID");
  }
 
  SDLoc DL(Op);
  static const Intrinsic::ID VssegInts[] = {
      Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
      Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
      Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
      Intrinsic::riscv_vsseg8_mask};
  static const Intrinsic::ID VsssegInts[] = {
      Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
      Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
      Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
      Intrinsic::riscv_vssseg8_mask};
 
  // Operands: (chain, int_id, vec*, ptr, mask, vl) or
  // (chain, int_id, vec*, ptr, stride, mask, vl)
  unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
  assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op->getOperand(2).getSimpleValueType();
  MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
  unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
                ContainerVT.getScalarSizeInBits();
  EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
 
  SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
  SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
  MVT MaskVT = Mask.getSimpleValueType();
  MVT MaskContainerVT = ::getContainerForFixedLengthVector(MaskVT, Subtarget);
  Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
 
  SDValue IntID = DAG.getTargetConstant(
      IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, XLenVT);
  SDValue Ptr = Op->getOperand(NF + 2);
 
  auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Op);
 
  SDValue StoredVal = DAG.getUNDEF(VecTupTy);
  for (unsigned i = 0; i < NF; i++)
    StoredVal = DAG.getNode(
        RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
        convertToScalableVector(ContainerVT, FixedIntrinsic->getOperand(2 + i),
                                DAG, Subtarget),
        DAG.getTargetConstant(i, DL, MVT::i32));
 
  SmallVector<SDValue, 10> Ops = {
      FixedIntrinsic->getChain(),
      IntID,
      StoredVal,
      Ptr,
      Mask,
      VL,
      DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
  // Insert the stride operand.
  if (IsStrided)
    Ops.insert(std::next(Ops.begin(), 4),
               Op.getOperand(Op.getNumOperands() - 3));
 
  return DAG.getMemIntrinsicNode(
      ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
      FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
}
 
SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
                                                 SelectionDAG &DAG) const {
  unsigned IntNo = Op.getConstantOperandVal(1);
  switch (IntNo) {
  default:
    break;
  case Intrinsic::riscv_seg2_store_mask:
  case Intrinsic::riscv_seg3_store_mask:
  case Intrinsic::riscv_seg4_store_mask:
  case Intrinsic::riscv_seg5_store_mask:
  case Intrinsic::riscv_seg6_store_mask:
  case Intrinsic::riscv_seg7_store_mask:
  case Intrinsic::riscv_seg8_store_mask:
  case Intrinsic::riscv_sseg2_store_mask:
  case Intrinsic::riscv_sseg3_store_mask:
  case Intrinsic::riscv_sseg4_store_mask:
  case Intrinsic::riscv_sseg5_store_mask:
  case Intrinsic::riscv_sseg6_store_mask:
  case Intrinsic::riscv_sseg7_store_mask:
  case Intrinsic::riscv_sseg8_store_mask:
    return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);
 
  case Intrinsic::riscv_sf_vc_xv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XV_SE);
  case Intrinsic::riscv_sf_vc_iv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IV_SE);
  case Intrinsic::riscv_sf_vc_vv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VV_SE);
  case Intrinsic::riscv_sf_vc_fv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FV_SE);
  case Intrinsic::riscv_sf_vc_xvv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XVV_SE);
  case Intrinsic::riscv_sf_vc_ivv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IVV_SE);
  case Intrinsic::riscv_sf_vc_vvv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VVV_SE);
  case Intrinsic::riscv_sf_vc_fvv_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FVV_SE);
  case Intrinsic::riscv_sf_vc_xvw_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XVW_SE);
  case Intrinsic::riscv_sf_vc_ivw_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IVW_SE);
  case Intrinsic::riscv_sf_vc_vvw_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VVW_SE);
  case Intrinsic::riscv_sf_vc_fvw_se:
    return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FVW_SE);
  }
 
  return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
}
 
static unsigned getRVVReductionOp(unsigned ISDOpcode) {
  switch (ISDOpcode) {
  default:
    llvm_unreachable("Unhandled reduction");
  case ISD::VP_REDUCE_ADD:
  case ISD::VECREDUCE_ADD:
    return RISCVISD::VECREDUCE_ADD_VL;
  case ISD::VP_REDUCE_UMAX:
  case ISD::VECREDUCE_UMAX:
    return RISCVISD::VECREDUCE_UMAX_VL;
  case ISD::VP_REDUCE_SMAX:
  case ISD::VECREDUCE_SMAX:
    return RISCVISD::VECREDUCE_SMAX_VL;
  case ISD::VP_REDUCE_UMIN:
  case ISD::VECREDUCE_UMIN:
    return RISCVISD::VECREDUCE_UMIN_VL;
  case ISD::VP_REDUCE_SMIN:
  case ISD::VECREDUCE_SMIN:
    return RISCVISD::VECREDUCE_SMIN_VL;
  case ISD::VP_REDUCE_AND:
  case ISD::VECREDUCE_AND:
    return RISCVISD::VECREDUCE_AND_VL;
  case ISD::VP_REDUCE_OR:
  case ISD::VECREDUCE_OR:
    return RISCVISD::VECREDUCE_OR_VL;
  case ISD::VP_REDUCE_XOR:
  case ISD::VECREDUCE_XOR:
    return RISCVISD::VECREDUCE_XOR_VL;
  case ISD::VP_REDUCE_FADD:
    return RISCVISD::VECREDUCE_FADD_VL;
  case ISD::VP_REDUCE_SEQ_FADD:
    return RISCVISD::VECREDUCE_SEQ_FADD_VL;
  case ISD::VP_REDUCE_FMAX:
  case ISD::VP_REDUCE_FMAXIMUM:
    return RISCVISD::VECREDUCE_FMAX_VL;
  case ISD::VP_REDUCE_FMIN:
  case ISD::VP_REDUCE_FMINIMUM:
    return RISCVISD::VECREDUCE_FMIN_VL;
  }
 
}
 
SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
                                                         SelectionDAG &DAG,
                                                         bool IsVP) const {
  SDLoc DL(Op);
  SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
  MVT VecVT = Vec.getSimpleValueType();
  assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
          Op.getOpcode() == ISD::VECREDUCE_OR ||
          Op.getOpcode() == ISD::VECREDUCE_XOR ||
          Op.getOpcode() == ISD::VP_REDUCE_AND ||
          Op.getOpcode() == ISD::VP_REDUCE_OR ||
          Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
         "Unexpected reduction lowering");
 
  MVT XLenVT = Subtarget.getXLenVT();
 
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
 
  SDValue Mask, VL;
  if (IsVP) {
    Mask = Op.getOperand(2);
    VL = Op.getOperand(3);
  } else {
    std::tie(Mask, VL) =
        getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
  }
 
  ISD::CondCode CC;
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unhandled reduction");
  case ISD::VECREDUCE_AND:
  case ISD::VP_REDUCE_AND: {
    // vcpop ~x == 0
    if (VecVT.isFixedLengthVector())
      Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec,
                        DAG.getAllOnesConstant(DL, ContainerVT), VL);
    else
      Vec = DAG.getNOT(DL, Vec, ContainerVT);
    Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
    CC = ISD::SETEQ;
    break;
  }
  case ISD::VECREDUCE_OR:
  case ISD::VP_REDUCE_OR:
    // vcpop x != 0
    Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
    CC = ISD::SETNE;
    break;
  case ISD::VECREDUCE_XOR:
  case ISD::VP_REDUCE_XOR: {
    // ((vcpop x) & 1) != 0
    SDValue One = DAG.getConstant(1, DL, XLenVT);
    Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
    Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
    CC = ISD::SETNE;
    break;
  }
  }
 
  SDValue Zero = DAG.getConstant(0, DL, XLenVT);
  SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
  SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC);
 
  if (!IsVP)
    return SetCC;
 
  // Now include the start value in the operation.
  // Note that we must return the start value when no elements are operated
  // upon. The vcpop instructions we've emitted in each case above will return
  // 0 for an inactive vector, and so we've already received the neutral value:
  // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
  // can simply include the start value.
  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
  return DAG.getNode(BaseOpc, DL, Op.getValueType(), SetCC, Op.getOperand(0));
}
 
static bool isNonZeroAVL(SDValue AVL) {
  auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
  auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
  return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
         (ImmAVL && ImmAVL->getZExtValue() >= 1);
}
 
/// Helper to lower a reduction sequence of the form:
/// scalar = reduce_op vec, scalar_start
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT,
                                 SDValue StartValue, SDValue Vec, SDValue Mask,
                                 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  const MVT VecVT = Vec.getSimpleValueType();
  const MVT M1VT = RISCVTargetLowering::getM1VT(VecVT);
  const MVT XLenVT = Subtarget.getXLenVT();
  const bool NonZeroAVL = isNonZeroAVL(VL);
 
  // The reduction needs an LMUL1 input; do the splat at either LMUL1
  // or the original VT if fractional.
  auto InnerVT = VecVT.bitsLE(M1VT) ? VecVT : M1VT;
  // We reuse the VL of the reduction to reduce vsetvli toggles if we can
  // prove it is non-zero.  For the AVL=0 case, we need the scalar to
  // be the result of the reduction operation.
  auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(1, DL, XLenVT);
  SDValue InitialValue =
      lowerScalarInsert(StartValue, InnerVL, InnerVT, DL, DAG, Subtarget);
  if (M1VT != InnerVT)
    InitialValue =
        DAG.getInsertSubvector(DL, DAG.getUNDEF(M1VT), InitialValue, 0);
  SDValue PassThru = NonZeroAVL ? DAG.getUNDEF(M1VT) : InitialValue;
  SDValue Policy = DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT);
  SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
  SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, Ops);
  return DAG.getExtractVectorElt(DL, ResVT, Reduction, 0);
}
 
SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
                                            SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Vec = Op.getOperand(0);
  EVT VecEVT = Vec.getValueType();
 
  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
 
  // Due to ordering in legalize types we may have a vector type that needs to
  // be split. Do that manually so we can get down to a legal type.
  while (getTypeAction(*DAG.getContext(), VecEVT) ==
         TargetLowering::TypeSplitVector) {
    auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
    VecEVT = Lo.getValueType();
    Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
  }
 
  // TODO: The type may need to be widened rather than split. Or widened before
  // it can be split.
  if (!isTypeLegal(VecEVT))
    return SDValue();
 
  MVT VecVT = VecEVT.getSimpleVT();
  MVT VecEltVT = VecVT.getVectorElementType();
  unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
 
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
  SDValue StartV;
  switch (BaseOpc) {
  default:
    StartV = DAG.getIdentityElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
    break;
  case ISD::AND:
  case ISD::OR:
  case ISD::UMAX:
  case ISD::UMIN:
  case ISD::SMAX:
  case ISD::SMIN:
    StartV = DAG.getExtractVectorElt(DL, VecEltVT, Vec, 0);
    break;
  }
  return lowerReductionSeq(RVVOpcode, Op.getSimpleValueType(), StartV, Vec,
                           Mask, VL, DL, DAG, Subtarget);
}
 
// Given a reduction op, this function returns the matching reduction opcode,
// the vector SDValue and the scalar SDValue required to lower this to a
// RISCVISD node.
static std::tuple<unsigned, SDValue, SDValue>
getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT,
                               const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  auto Flags = Op->getFlags();
  unsigned Opcode = Op.getOpcode();
  switch (Opcode) {
  default:
    llvm_unreachable("Unhandled reduction");
  case ISD::VECREDUCE_FADD: {
    // Use positive zero if we can. It is cheaper to materialize.
    SDValue Zero =
        DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
    return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
  }
  case ISD::VECREDUCE_SEQ_FADD:
    return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
                           Op.getOperand(0));
  case ISD::VECREDUCE_FMINIMUM:
  case ISD::VECREDUCE_FMAXIMUM:
  case ISD::VECREDUCE_FMIN:
  case ISD::VECREDUCE_FMAX: {
    SDValue Front = DAG.getExtractVectorElt(DL, EltVT, Op.getOperand(0), 0);
    unsigned RVVOpc =
        (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
            ? RISCVISD::VECREDUCE_FMIN_VL
            : RISCVISD::VECREDUCE_FMAX_VL;
    return std::make_tuple(RVVOpc, Op.getOperand(0), Front);
  }
  }
}
 
SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VecEltVT = Op.getSimpleValueType();
 
  unsigned RVVOpcode;
  SDValue VectorVal, ScalarVal;
  std::tie(RVVOpcode, VectorVal, ScalarVal) =
      getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT, Subtarget);
  MVT VecVT = VectorVal.getSimpleValueType();
 
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
  }
 
  MVT ResVT = Op.getSimpleValueType();
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
  SDValue Res = lowerReductionSeq(RVVOpcode, ResVT, ScalarVal, VectorVal, Mask,
                                  VL, DL, DAG, Subtarget);
  if (Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
      Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
    return Res;
 
  if (Op->getFlags().hasNoNaNs())
    return Res;
 
  // Force output to NaN if any element is Nan.
  SDValue IsNan =
      DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                  {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
                   DAG.getUNDEF(Mask.getValueType()), Mask, VL});
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue CPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNan, Mask, VL);
  SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, CPop,
                                DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
  return DAG.getSelect(
      DL, ResVT, NoNaNs, Res,
      DAG.getConstantFP(APFloat::getNaN(ResVT.getFltSemantics()), DL, ResVT));
}
 
SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
                                           SelectionDAG &DAG) const {
  SDLoc DL(Op);
  unsigned Opc = Op.getOpcode();
  SDValue Start = Op.getOperand(0);
  SDValue Vec = Op.getOperand(1);
  EVT VecEVT = Vec.getValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  // TODO: The type may need to be widened rather than split. Or widened before
  // it can be split.
  if (!isTypeLegal(VecEVT))
    return SDValue();
 
  MVT VecVT = VecEVT.getSimpleVT();
  unsigned RVVOpcode = getRVVReductionOp(Opc);
 
  SDValue VL = Op.getOperand(3);
  SDValue Mask = Op.getOperand(2);
  if (VecVT.isFixedLengthVector()) {
    auto ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
    Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
                                   Subtarget);
  }
  SDValue Res =
      lowerReductionSeq(RVVOpcode, Op.getSimpleValueType(), Op.getOperand(0),
                        Vec, Mask, VL, DL, DAG, Subtarget);
  if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
      Op->getFlags().hasNoNaNs())
    return Res;
 
  // Propagate NaNs.
  MVT PredVT = getMaskTypeFor(Vec.getSimpleValueType());
  // Check if any of the elements in Vec is NaN.
  SDValue IsNaN = DAG.getNode(
      RISCVISD::SETCC_VL, DL, PredVT,
      {Vec, Vec, DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(PredVT), Mask, VL});
  SDValue VCPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNaN, Mask, VL);
  // Check if the start value is NaN.
  SDValue StartIsNaN = DAG.getSetCC(DL, XLenVT, Start, Start, ISD::SETUO);
  VCPop = DAG.getNode(ISD::OR, DL, XLenVT, VCPop, StartIsNaN);
  SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, VCPop,
                                DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
  MVT ResVT = Res.getSimpleValueType();
  return DAG.getSelect(
      DL, ResVT, NoNaNs, Res,
      DAG.getConstantFP(APFloat::getNaN(ResVT.getFltSemantics()), DL, ResVT));
}
 
static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
                                          SDValue V, MVT WideVT);
 
SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
                                                   SelectionDAG &DAG) const {
  SDValue Vec = Op.getOperand(0);
  SDValue SubVec = Op.getOperand(1);
  MVT VecVT = Vec.getSimpleValueType();
  MVT SubVecVT = SubVec.getSimpleValueType();
 
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned OrigIdx = Op.getConstantOperandVal(2);
  const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
 
  bool IsPExtInsert =
      Subtarget.hasStdExtP() &&
      ((Subtarget.is64Bit() &&
        (SubVecVT == MVT::v2i16 || SubVecVT == MVT::v4i8)) ||
       (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)));
 
  // Fold insert of a 32-bit packed type into a zero-filled 64-bit packed vector
  // at index 0 (a zero-extend) to avoid scalarizing it into a byte-wise repack.
  if (IsPExtInsert) {
    if ((VecVT != MVT::v4i16 && VecVT != MVT::v8i8) ||
        SubVecVT.getSizeInBits() != 32 || OrigIdx != 0 ||
        !ISD::isConstantSplatVectorAllZeros(Vec.getNode()))
      return SDValue();
 
    if (!Subtarget.is64Bit()) {
      SDValue Zero = DAG.getBitcast(SubVecVT, DAG.getConstant(0, DL, MVT::i32));
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, SubVec, Zero);
    }
    return widenPackedVectorWithZeros(DAG, DL, SubVec, VecVT);
  }
 
  if (OrigIdx == 0 && Vec.isUndef())
    return Op;
 
  // We don't have the ability to slide mask vectors up indexed by their i1
  // elements; the smallest we can do is i8. Often we are able to bitcast to
  // equivalent i8 vectors. Note that when inserting a fixed-length vector
  // into a scalable one, we might not necessarily have enough scalable
  // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
  if (SubVecVT.getVectorElementType() == MVT::i1) {
    if (VecVT.getVectorMinNumElements() >= 8 &&
        SubVecVT.getVectorMinNumElements() >= 8) {
      assert(OrigIdx % 8 == 0 && "Invalid index");
      assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
             SubVecVT.getVectorMinNumElements() % 8 == 0 &&
             "Unexpected mask vector lowering");
      OrigIdx /= 8;
      SubVecVT =
          MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
                           SubVecVT.isScalableVector());
      VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
                               VecVT.isScalableVector());
      Vec = DAG.getBitcast(VecVT, Vec);
      SubVec = DAG.getBitcast(SubVecVT, SubVec);
    } else {
      // We can't slide this mask vector up indexed by its i1 elements.
      // This poses a problem when we wish to insert a scalable vector which
      // can't be re-expressed as a larger type. Just choose the slow path and
      // extend to a larger type, then truncate back down.
      MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
      MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
      Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
      SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
      Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
                        Op.getOperand(2));
      SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
      return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
    }
  }
 
  // If the subvector vector is a fixed-length type and we don't know VLEN
  // exactly, we cannot use subregister manipulation to simplify the codegen; we
  // don't know which register of a LMUL group contains the specific subvector
  // as we only know the minimum register size. Therefore we must slide the
  // vector group up the full amount.
  const auto VLen = Subtarget.getRealVLen();
  if (SubVecVT.isFixedLengthVector() && !VLen) {
    MVT ContainerVT = VecVT;
    if (VecVT.isFixedLengthVector()) {
      ContainerVT = getContainerForFixedLengthVector(VecVT);
      Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
    }
 
    SubVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), SubVec, 0);
 
    SDValue Mask =
        getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
    // Set the vector length to only the number of elements we care about. Note
    // that for slideup this includes the offset.
    unsigned EndIndex = OrigIdx + SubVecVT.getVectorNumElements();
    SDValue VL = DAG.getConstant(EndIndex, DL, XLenVT);
 
    // Use tail agnostic policy if we're inserting over Vec's tail.
    unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
    if (VecVT.isFixedLengthVector() && EndIndex == VecVT.getVectorNumElements())
      Policy = RISCVVType::TAIL_AGNOSTIC;
 
    // If we're inserting into the lowest elements, use a tail undisturbed
    // vmv.v.v.
    if (OrigIdx == 0) {
      SubVec =
          DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL);
    } else {
      SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
      SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec,
                           SlideupAmt, Mask, VL, Policy);
    }
 
    if (VecVT.isFixedLengthVector())
      SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
    return DAG.getBitcast(Op.getValueType(), SubVec);
  }
 
  MVT ContainerVecVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVecVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVecVT, Vec, DAG, Subtarget);
  }
 
  MVT ContainerSubVecVT = SubVecVT;
  if (SubVecVT.isFixedLengthVector()) {
    ContainerSubVecVT = getContainerForFixedLengthVector(SubVecVT);
    SubVec = convertToScalableVector(ContainerSubVecVT, SubVec, DAG, Subtarget);
  }
 
  unsigned SubRegIdx;
  ElementCount RemIdx;
  // insert_subvector scales the index by vscale if the subvector is scalable,
  // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
  // we have a fixed length subvector, we need to adjust the index by 1/vscale.
  if (SubVecVT.isFixedLengthVector()) {
    assert(VLen);
    unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
    auto Decompose =
        RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
            ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale, TRI);
    SubRegIdx = Decompose.first;
    RemIdx = ElementCount::getFixed((Decompose.second * Vscale) +
                                    (OrigIdx % Vscale));
  } else {
    auto Decompose =
        RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
            ContainerVecVT, ContainerSubVecVT, OrigIdx, TRI);
    SubRegIdx = Decompose.first;
    RemIdx = ElementCount::getScalable(Decompose.second);
  }
 
  TypeSize VecRegSize = TypeSize::getScalable(RISCV::RVVBitsPerBlock);
  assert(isPowerOf2_64(
      Subtarget.expandVScale(SubVecVT.getSizeInBits()).getKnownMinValue()));
  bool ExactlyVecRegSized =
      Subtarget.expandVScale(SubVecVT.getSizeInBits())
          .isKnownMultipleOf(Subtarget.expandVScale(VecRegSize));
 
  // 1. If the Idx has been completely eliminated and this subvector's size is
  // a vector register or a multiple thereof, or the surrounding elements are
  // undef, then this is a subvector insert which naturally aligns to a vector
  // register. These can easily be handled using subregister manipulation.
  // 2. If the subvector isn't an exact multiple of a valid register group size,
  // then the insertion must preserve the undisturbed elements of the register.
  // We do this by lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1
  // vector type (which resolves to a subregister copy), performing a VSLIDEUP
  // to place the subvector within the vector register, and an INSERT_SUBVECTOR
  // of that LMUL=1 type back into the larger vector (resolving to another
  // subregister operation). See below for how our VSLIDEUP works. We go via a
  // LMUL=1 type to avoid allocating a large register group to hold our
  // subvector.
  if (RemIdx.isZero() && (ExactlyVecRegSized || Vec.isUndef())) {
    if (SubVecVT.isFixedLengthVector()) {
      // We may get NoSubRegister if inserting at index 0 and the subvec
      // container is the same as the vector, e.g. vec=v4i32,subvec=v4i32,idx=0
      if (SubRegIdx == RISCV::NoSubRegister) {
        assert(OrigIdx == 0);
        return Op;
      }
 
      // Use a insert_subvector that will resolve to an insert subreg.
      assert(VLen);
      unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
      SDValue Insert =
          DAG.getInsertSubvector(DL, Vec, SubVec, OrigIdx / Vscale);
      if (VecVT.isFixedLengthVector())
        Insert = convertFromScalableVector(VecVT, Insert, DAG, Subtarget);
      return Insert;
    }
    return Op;
  }
 
  // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
  // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
  // (in our case undisturbed). This means we can set up a subvector insertion
  // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
  // size of the subvector.
  MVT InterSubVT = ContainerVecVT;
  SDValue AlignedExtract = Vec;
  unsigned AlignedIdx = OrigIdx - RemIdx.getKnownMinValue();
  if (SubVecVT.isFixedLengthVector()) {
    assert(VLen);
    AlignedIdx /= *VLen / RISCV::RVVBitsPerBlock;
  }
  if (ContainerVecVT.bitsGT(RISCVTargetLowering::getM1VT(ContainerVecVT))) {
    InterSubVT = RISCVTargetLowering::getM1VT(ContainerVecVT);
    // Extract a subvector equal to the nearest full vector register type. This
    // should resolve to a EXTRACT_SUBREG instruction.
    AlignedExtract = DAG.getExtractSubvector(DL, InterSubVT, Vec, AlignedIdx);
  }
 
  SubVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(InterSubVT), SubVec, 0);
 
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
 
  ElementCount EndIndex = RemIdx + SubVecVT.getVectorElementCount();
  VL = DAG.getElementCount(DL, XLenVT, SubVecVT.getVectorElementCount());
 
  // Use tail agnostic policy if we're inserting over InterSubVT's tail.
  unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
  if (Subtarget.expandVScale(EndIndex) ==
      Subtarget.expandVScale(InterSubVT.getVectorElementCount()))
    Policy = RISCVVType::TAIL_AGNOSTIC;
 
  // If we're inserting into the lowest elements, use a tail undisturbed
  // vmv.v.v.
  if (RemIdx.isZero()) {
    SubVec = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, InterSubVT, AlignedExtract,
                         SubVec, VL);
  } else {
    SDValue SlideupAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
 
    // Construct the vector length corresponding to RemIdx + length(SubVecVT).
    VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
 
    SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec,
                         SlideupAmt, Mask, VL, Policy);
  }
 
  // If required, insert this subvector back into the correct vector register.
  // This should resolve to an INSERT_SUBREG instruction.
  if (ContainerVecVT.bitsGT(InterSubVT))
    SubVec = DAG.getInsertSubvector(DL, Vec, SubVec, AlignedIdx);
 
  if (VecVT.isFixedLengthVector())
    SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
 
  // We might have bitcast from a mask type: cast back to the original type if
  // required.
  return DAG.getBitcast(Op.getSimpleValueType(), SubVec);
}
 
SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
                                                    SelectionDAG &DAG) const {
  SDValue Vec = Op.getOperand(0);
  MVT SubVecVT = Op.getSimpleValueType();
  MVT VecVT = Vec.getSimpleValueType();
 
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned OrigIdx = Op.getConstantOperandVal(1);
  const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
 
  // With an index of 0 this is a cast-like subvector, which can be performed
  // with subregister operations.
  if (OrigIdx == 0)
    return Op;
 
  // We don't have the ability to slide mask vectors down indexed by their i1
  // elements; the smallest we can do is i8. Often we are able to bitcast to
  // equivalent i8 vectors. Note that when extracting a fixed-length vector
  // from a scalable one, we might not necessarily have enough scalable
  // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
  if (SubVecVT.getVectorElementType() == MVT::i1) {
    if (VecVT.getVectorMinNumElements() >= 8 &&
        SubVecVT.getVectorMinNumElements() >= 8) {
      assert(OrigIdx % 8 == 0 && "Invalid index");
      assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
             SubVecVT.getVectorMinNumElements() % 8 == 0 &&
             "Unexpected mask vector lowering");
      OrigIdx /= 8;
      SubVecVT =
          MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
                           SubVecVT.isScalableVector());
      VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
                               VecVT.isScalableVector());
      Vec = DAG.getBitcast(VecVT, Vec);
    } else {
      // We can't slide this mask vector down, indexed by its i1 elements.
      // This poses a problem when we wish to extract a scalable vector which
      // can't be re-expressed as a larger type. Just choose the slow path and
      // extend to a larger type, then truncate back down.
      // TODO: We could probably improve this when extracting certain fixed
      // from fixed, where we can extract as i8 and shift the correct element
      // right to reach the desired subvector?
      MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
      MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
      Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
      Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
                        Op.getOperand(1));
      SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
      return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
    }
  }
 
  const auto VLen = Subtarget.getRealVLen();
 
  // If the subvector vector is a fixed-length type and we don't know VLEN
  // exactly, we cannot use subregister manipulation to simplify the codegen; we
  // don't know which register of a LMUL group contains the specific subvector
  // as we only know the minimum register size. Therefore we must slide the
  // vector group down the full amount.
  if (SubVecVT.isFixedLengthVector() && !VLen) {
    MVT ContainerVT = VecVT;
    if (VecVT.isFixedLengthVector()) {
      ContainerVT = getContainerForFixedLengthVector(VecVT);
      Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
    }
 
    // Shrink down Vec so we're performing the slidedown on a smaller LMUL.
    unsigned LastIdx = OrigIdx + SubVecVT.getVectorNumElements() - 1;
    if (auto ShrunkVT =
            getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) {
      ContainerVT = *ShrunkVT;
      Vec = DAG.getExtractSubvector(DL, ContainerVT, Vec, 0);
    }
 
    SDValue Mask =
        getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
    // Set the vector length to only the number of elements we care about. This
    // avoids sliding down elements we're going to discard straight away.
    SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
    SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
    SDValue Slidedown =
        getVSlidedown(DAG, Subtarget, DL, ContainerVT,
                      DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
    // Now we can use a cast-like subvector extract to get the result.
    Slidedown = DAG.getExtractSubvector(DL, SubVecVT, Slidedown, 0);
    return DAG.getBitcast(Op.getValueType(), Slidedown);
  }
 
  if (VecVT.isFixedLengthVector()) {
    VecVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(VecVT, Vec, DAG, Subtarget);
  }
 
  MVT ContainerSubVecVT = SubVecVT;
  if (SubVecVT.isFixedLengthVector())
    ContainerSubVecVT = getContainerForFixedLengthVector(SubVecVT);
 
  unsigned SubRegIdx;
  ElementCount RemIdx;
  // extract_subvector scales the index by vscale if the subvector is scalable,
  // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
  // we have a fixed length subvector, we need to adjust the index by 1/vscale.
  if (SubVecVT.isFixedLengthVector()) {
    assert(VLen);
    unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
    auto Decompose =
        RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
            VecVT, ContainerSubVecVT, OrigIdx / Vscale, TRI);
    SubRegIdx = Decompose.first;
    RemIdx = ElementCount::getFixed((Decompose.second * Vscale) +
                                    (OrigIdx % Vscale));
  } else {
    auto Decompose =
        RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
            VecVT, ContainerSubVecVT, OrigIdx, TRI);
    SubRegIdx = Decompose.first;
    RemIdx = ElementCount::getScalable(Decompose.second);
  }
 
  // If the Idx has been completely eliminated then this is a subvector extract
  // which naturally aligns to a vector register. These can easily be handled
  // using subregister manipulation. We use an extract_subvector that will
  // resolve to an extract subreg.
  if (RemIdx.isZero()) {
    if (SubVecVT.isFixedLengthVector()) {
      assert(VLen);
      unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
      Vec =
          DAG.getExtractSubvector(DL, ContainerSubVecVT, Vec, OrigIdx / Vscale);
      return convertFromScalableVector(SubVecVT, Vec, DAG, Subtarget);
    }
    return Op;
  }
 
  // Else SubVecVT is M1 or smaller and may need to be slid down: if SubVecVT
  // was > M1 then the index would need to be a multiple of VLMAX, and so would
  // divide exactly.
  assert(RISCVVType::decodeVLMUL(getLMUL(ContainerSubVecVT)).second ||
         getLMUL(ContainerSubVecVT) == RISCVVType::LMUL_1);
 
  // If the vector type is an LMUL-group type, extract a subvector equal to the
  // nearest full vector register type.
  MVT InterSubVT = VecVT;
  if (VecVT.bitsGT(RISCVTargetLowering::getM1VT(VecVT))) {
    // If VecVT has an LMUL > 1, then SubVecVT should have a smaller LMUL, and
    // we should have successfully decomposed the extract into a subregister.
    // We use an extract_subvector that will resolve to a subreg extract.
    assert(SubRegIdx != RISCV::NoSubRegister);
    (void)SubRegIdx;
    unsigned Idx = OrigIdx - RemIdx.getKnownMinValue();
    if (SubVecVT.isFixedLengthVector()) {
      assert(VLen);
      Idx /= *VLen / RISCV::RVVBitsPerBlock;
    }
    InterSubVT = RISCVTargetLowering::getM1VT(VecVT);
    Vec = DAG.getExtractSubvector(DL, InterSubVT, Vec, Idx);
  }
 
  // Slide this vector register down by the desired number of elements in order
  // to place the desired subvector starting at element 0.
  SDValue SlidedownAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
  auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
  if (SubVecVT.isFixedLengthVector())
    VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
  SDValue Slidedown =
      getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT),
                    Vec, SlidedownAmt, Mask, VL);
 
  // Now the vector is in the right position, extract our final subvector. This
  // should resolve to a COPY.
  Slidedown = DAG.getExtractSubvector(DL, SubVecVT, Slidedown, 0);
 
  // We might have bitcast from a mask type: cast back to the original type if
  // required.
  return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
}
 
// Widen a vector's operands to i8, then truncate its results back to the
// original type, typically i1.  All operand and result types must be the same.
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL,
                                  SelectionDAG &DAG) {
  MVT VT = N.getSimpleValueType();
  MVT WideVT = VT.changeVectorElementType(MVT::i8);
  SmallVector<SDValue, 4> WideOps;
  for (SDValue Op : N->ops()) {
    assert(Op.getSimpleValueType() == VT &&
           "Operands and result must be same type");
    WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op));
  }
 
  unsigned NumVals = N->getNumValues();
 
  SDVTList VTs = DAG.getVTList(SmallVector<EVT, 4>(
      NumVals,
      N.getValueType().changeVectorElementType(*DAG.getContext(), MVT::i8)));
  SDValue WideN = DAG.getNode(N.getOpcode(), DL, VTs, WideOps);
  SmallVector<SDValue, 4> TruncVals;
  for (unsigned I = 0; I < NumVals; I++) {
    TruncVals.push_back(
        DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I),
                     DAG.getConstant(0, DL, WideVT), ISD::SETNE));
  }
 
  if (TruncVals.size() > 1)
    return DAG.getMergeValues(TruncVals, DL);
  return TruncVals.front();
}
 
SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
                                                      SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VecVT = Op.getSimpleValueType();
 
  const unsigned Factor = Op->getNumValues();
  assert(Factor <= 8);
 
  // 1 bit element vectors need to be widened to e8
  if (VecVT.getVectorElementType() == MVT::i1)
    return widenVectorOpsToi8(Op, DL, DAG);
 
  bool IsFixedVector = VecVT.isFixedLengthVector();
 
  MVT ContainerVecVT = VecVT;
  if (IsFixedVector)
    ContainerVecVT = getContainerForFixedLengthVector(VecVT);
 
  // If concatenating would exceed LMUL=8, we need to split.
  if ((ContainerVecVT.getSizeInBits().getKnownMinValue() * Factor) >
      (8 * RISCV::RVVBitsPerBlock)) {
    SmallVector<SDValue, 8> Ops(Factor * 2);
    for (unsigned i = 0; i != Factor; ++i) {
      auto [OpLo, OpHi] = DAG.SplitVectorOperand(Op.getNode(), i);
      Ops[i * 2] = OpLo;
      Ops[i * 2 + 1] = OpHi;
    }
 
    SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
 
    SDValue Lo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
                             ArrayRef(Ops).slice(0, Factor));
    SDValue Hi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
                             ArrayRef(Ops).slice(Factor, Factor));
 
    SmallVector<SDValue, 8> Res(Factor);
    for (unsigned i = 0; i != Factor; ++i)
      Res[i] = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo.getValue(i),
                           Hi.getValue(i));
 
    return DAG.getMergeValues(Res, DL);
  }
 
  if (Subtarget.hasStdExtZvzip() && Factor == 2 && !IsFixedVector) {
    MVT VT = Op->getSimpleValueType(0);
    MVT NewVT = VT.getDoubleNumVectorElementsVT();
    if (isTypeLegal(NewVT) &&
        isLegalVTForZvzipDeinterleavedOperand(VT, Subtarget)) {
      SDValue V1 = Op->getOperand(0);
      SDValue V2 = Op->getOperand(1);
      SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
      SDValue Even =
          lowerZvzipVUNZIP(RISCVISD::VUNZIPE_VL, V, DL, DAG, Subtarget);
      SDValue Odd =
          lowerZvzipVUNZIP(RISCVISD::VUNZIPO_VL, V, DL, DAG, Subtarget);
      return DAG.getMergeValues({Even, Odd}, DL);
    }
  }
 
  SmallVector<SDValue, 8> Ops(Op->op_values());
 
  // Concatenate the vectors as one vector to deinterleave
  MVT ConcatVT =
      MVT::getVectorVT(VecVT.getVectorElementType(),
                       VecVT.getVectorElementCount() * PowerOf2Ceil(Factor));
  if (Ops.size() < PowerOf2Ceil(Factor))
    Ops.append(PowerOf2Ceil(Factor) - Factor, DAG.getUNDEF(VecVT));
  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Ops);
 
  if (Factor == 2 && !IsFixedVector) {
    // We can deinterleave through vnsrl.wi if the element type is smaller than
    // ELEN
    if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
      SDValue Even = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 0, DAG);
      SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 1, DAG);
      return DAG.getMergeValues({Even, Odd}, DL);
    }
 
    // For the indices, use the vmv.v.x of an i8 constant to fill the largest
    // possibly mask vector, then extract the required subvector.  Doing this
    // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
    // creation to be rematerialized during register allocation to reduce
    // register pressure if needed.
 
    MVT MaskVT = ConcatVT.changeVectorElementType(MVT::i1);
 
    SDValue EvenSplat = DAG.getConstant(0b01010101, DL, MVT::nxv8i8);
    EvenSplat = DAG.getBitcast(MVT::nxv64i1, EvenSplat);
    SDValue EvenMask = DAG.getExtractSubvector(DL, MaskVT, EvenSplat, 0);
 
    SDValue OddSplat = DAG.getConstant(0b10101010, DL, MVT::nxv8i8);
    OddSplat = DAG.getBitcast(MVT::nxv64i1, OddSplat);
    SDValue OddMask = DAG.getExtractSubvector(DL, MaskVT, OddSplat, 0);
 
    // vcompress the even and odd elements into two separate vectors
    SDValue EvenWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
                                   EvenMask, DAG.getUNDEF(ConcatVT));
    SDValue OddWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
                                  OddMask, DAG.getUNDEF(ConcatVT));
 
    // Extract the result half of the gather for even and odd
    SDValue Even = DAG.getExtractSubvector(DL, VecVT, EvenWide, 0);
    SDValue Odd = DAG.getExtractSubvector(DL, VecVT, OddWide, 0);
 
    return DAG.getMergeValues({Even, Odd}, DL);
  }
 
  // Store with unit-stride store and load it back with segmented load.
  SDValue Mask, VL;
  MVT XLenVT = Subtarget.getXLenVT();
  auto &MF = DAG.getMachineFunction();
  SDValue Chain = DAG.getEntryNode();
  Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
  SDValue StackPtr;
  MachinePointerInfo PtrInfo;
  if (IsFixedVector) {
    // Calculating the stack size.
    ElementCount ActualConcatEC = VecVT.getVectorElementCount() * Factor;
    EVT ConcatEVT = EVT::getVectorVT(
        *DAG.getContext(), VecVT.getVectorElementType(), ActualConcatEC);
    StackPtr = DAG.CreateStackTemporary(ConcatEVT.getStoreSize(), Alignment);
    auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
 
    // If this is a fixed vector, instead of using the concat vector, we simply
    // store each fixed vector operand directly onto the stack, individually.
    // The reason being that if the fixed vector is (much) smaller than the
    // container vector, we will be wasting space on stack.
    TypeSize VecSize = VecVT.getStoreSize();
    SDValue BasePtr = StackPtr;
    MachinePointerInfo PI = PtrInfo;
    SmallVector<SDValue, 8> Tokens(Factor);
    for (auto [Idx, FieldOp] : enumerate(Op->op_values())) {
      if (Idx) {
        // Advance the pointer.
        BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, VecSize);
        PI = PI.getWithOffset(VecSize);
      }
      Tokens[Idx] = DAG.getStore(Chain, DL, FieldOp, BasePtr, PI, Alignment);
    }
    Chain = DAG.getTokenFactor(DL, Tokens);
 
    // Calculating Mask and VL for later usages.
    std::tie(Mask, VL) =
        getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
    ConcatVT = getContainerForFixedLengthVector(ConcatVT);
  } else {
    std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
    StackPtr = DAG.CreateStackTemporary(ConcatVT.getStoreSize(), Alignment);
    auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
 
    SDValue StoreOps[] = {
        Chain, DAG.getTargetConstant(Intrinsic::riscv_vse, DL, XLenVT), Concat,
        StackPtr, VL};
 
    Chain = DAG.getMemIntrinsicNode(
        ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), StoreOps,
        ConcatVT.getVectorElementType(), PtrInfo, Alignment,
        MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer());
  }
 
  // Load it back with segmented load.
  SDValue Passthru = DAG.getUNDEF(ConcatVT);
  static const Intrinsic::ID VlsegIntrinsicsIds[] = {
      Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
      Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
      Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
      Intrinsic::riscv_vlseg8_mask};
 
  SDValue LoadOps[] = {
      Chain,
      DAG.getTargetConstant(VlsegIntrinsicsIds[Factor - 2], DL, XLenVT),
      Passthru,
      StackPtr,
      Mask,
      VL,
      DAG.getTargetConstant(
          RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, XLenVT),
      DAG.getTargetConstant(Log2_64(VecVT.getScalarSizeInBits()), DL, XLenVT)};
 
  unsigned Sz = Factor * ContainerVecVT.getVectorMinNumElements() *
                ContainerVecVT.getScalarSizeInBits();
  EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, Factor);
 
  SDValue Load = DAG.getMemIntrinsicNode(
      ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList({VecTupTy, MVT::Other}),
      LoadOps, ConcatVT.getVectorElementType(), PtrInfo, Alignment,
      MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer());
 
  SmallVector<SDValue, 8> Res(Factor);
 
  for (unsigned i = 0U; i < Factor; ++i) {
    SDValue FieldRes =
        DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, ContainerVecVT, Load,
                    DAG.getTargetConstant(i, DL, MVT::i32));
    if (IsFixedVector)
      FieldRes = convertFromScalableVector(VecVT, FieldRes, DAG, Subtarget);
    Res[i] = FieldRes;
  }
 
  return DAG.getMergeValues(Res, DL);
}
 
SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
                                                    SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VecVT = Op.getSimpleValueType();
 
  const unsigned Factor = Op.getNumOperands();
  assert(Factor <= 8);
 
  // i1 vectors need to be widened to i8
  if (VecVT.getVectorElementType() == MVT::i1)
    return widenVectorOpsToi8(Op, DL, DAG);
 
  MVT ContainerVecVT = VecVT;
  if (VecVT.isFixedLengthVector())
    ContainerVecVT = getContainerForFixedLengthVector(VecVT);
 
  // If the VT is larger than LMUL=8, we need to split and reassemble.
  if ((ContainerVecVT.getSizeInBits().getKnownMinValue() * Factor) >
          (8 * RISCV::RVVBitsPerBlock) ||
      // In an unlikely case, a vector type might be legal to RISC-V but
      // exceeds the MVT limit, in which we also want to split it.
      (Subtarget.hasStdExtZvzip() && Factor == 2 &&
       !isTypeLegal(VecVT.changeVectorElementCount(
           VecVT.getVectorElementCount() * Factor)))) {
    SmallVector<SDValue, 8> Ops(Factor * 2);
    for (unsigned i = 0; i != Factor; ++i) {
      auto [OpLo, OpHi] = DAG.SplitVectorOperand(Op.getNode(), i);
      Ops[i] = OpLo;
      Ops[i + Factor] = OpHi;
    }
 
    SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
 
    SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
                                 ArrayRef(Ops).take_front(Factor)),
                     DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
                                 ArrayRef(Ops).drop_front(Factor))};
 
    SmallVector<SDValue, 8> Concats(Factor);
    for (unsigned i = 0; i != Factor; ++i) {
      unsigned IdxLo = 2 * i;
      unsigned IdxHi = 2 * i + 1;
      Concats[i] = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
                               Res[IdxLo / Factor].getValue(IdxLo % Factor),
                               Res[IdxHi / Factor].getValue(IdxHi % Factor));
    }
 
    return DAG.getMergeValues(Concats, DL);
  }
 
  MVT XLenVT = Subtarget.getXLenVT();
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
 
  SDValue Interleaved;
 
  // Spill to the stack using a segment store for simplicity.
  if (Factor != 2) {
    EVT MemVT =
        EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(),
                         VecVT.getVectorElementCount() * Factor);
 
    // Allocate a stack slot.
    // Note that in the case where VecVT is fixed vector, even we later
    // create a container for each (fixed vector) operand, we still allocate
    // the stack with fixed vector size, rather than container size, as
    // fixed vector size is already sufficient.
    Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
    SDValue StackPtr =
        DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
    EVT PtrVT = StackPtr.getValueType();
    auto &MF = DAG.getMachineFunction();
    auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
    auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
 
    static const Intrinsic::ID IntrIds[] = {
        Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
        Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
        Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
        Intrinsic::riscv_vsseg8_mask,
    };
 
    unsigned Sz = Factor * ContainerVecVT.getVectorMinNumElements() *
                  ContainerVecVT.getScalarSizeInBits();
    EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, Factor);
 
    SDValue StoredVal = DAG.getUNDEF(VecTupTy);
    for (unsigned i = 0; i < Factor; i++) {
      SDValue OpVal = Op.getOperand(i);
      if (VecVT.isFixedLengthVector())
        OpVal = convertToScalableVector(ContainerVecVT, OpVal, DAG, Subtarget);
      StoredVal = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
                              OpVal, DAG.getTargetConstant(i, DL, MVT::i32));
    }
 
    SDValue Ops[] = {DAG.getEntryNode(),
                     DAG.getTargetConstant(IntrIds[Factor - 2], DL, XLenVT),
                     StoredVal,
                     StackPtr,
                     Mask,
                     VL,
                     DAG.getTargetConstant(Log2_64(VecVT.getScalarSizeInBits()),
                                           DL, XLenVT)};
 
    SDValue Chain = DAG.getMemIntrinsicNode(
        ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
        VecVT.getVectorElementType(), PtrInfo, Alignment,
        MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer());
 
    SmallVector<SDValue, 8> Loads(Factor);
 
    SDValue Increment = DAG.getTypeSize(DL, PtrVT, VecVT.getStoreSize());
    for (unsigned i = 0; i != Factor; ++i) {
      if (i != 0)
        StackPtr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, Increment);
 
      Loads[i] = DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo);
    }
 
    return DAG.getMergeValues(Loads, DL);
  }
 
  if (Subtarget.hasStdExtZvzip() && !Op.getOperand(0).isUndef() &&
      !Op.getOperand(1).isUndef()) {
    MVT VT = Op->getSimpleValueType(0);
    MVT NewVT = VT.getDoubleNumVectorElementsVT();
    if (isLegalVTForZvzipInterleavedOperand(NewVT, Subtarget)) {
      // Freeze the sources so we can increase their use count.
      SDValue V1 = DAG.getFreeze(Op->getOperand(0));
      SDValue V2 = DAG.getFreeze(Op->getOperand(1));
      SDValue Interleaved = lowerZvzipVZIP(V1, V2, DL, DAG, Subtarget);
      SDValue Lo = DAG.getExtractSubvector(DL, VT, Interleaved, 0);
      SDValue Hi = DAG.getExtractSubvector(DL, VT, Interleaved,
                                           VT.getVectorMinNumElements());
      return DAG.getMergeValues({Lo, Hi}, DL);
    }
  }
 
  // If the element type is smaller than ELEN, then we can interleave with
  // vwaddu.vv and vwmaccu.vx
  if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
    Interleaved = getWideningInterleave(Op.getOperand(0), Op.getOperand(1), DL,
                                        DAG, Subtarget);
  } else {
    // Otherwise, fallback to using vrgathere16.vv
    MVT ConcatVT = MVT::getVectorVT(VecVT.getVectorElementType(),
                                    VecVT.getVectorElementCount() * 2);
    SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
                                 Op.getOperand(0), Op.getOperand(1));
 
    MVT IdxVT = ConcatVT.changeVectorElementType(MVT::i16);
 
    // 0 1 2 3 4 5 6 7 ...
    SDValue StepVec = DAG.getStepVector(DL, IdxVT);
 
    // 1 1 1 1 1 1 1 1 ...
    SDValue Ones = DAG.getSplatVector(IdxVT, DL, DAG.getConstant(1, DL, XLenVT));
 
    // 1 0 1 0 1 0 1 0 ...
    SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones);
    OddMask = DAG.getSetCC(
        DL, IdxVT.changeVectorElementType(MVT::i1), OddMask,
        DAG.getSplatVector(IdxVT, DL, DAG.getConstant(0, DL, XLenVT)),
        ISD::CondCode::SETNE);
 
    SDValue VLMax = DAG.getSplatVector(IdxVT, DL, computeVLMax(VecVT, DL, DAG));
 
    // Build up the index vector for interleaving the concatenated vector
    //      0      0      1      1      2      2      3      3 ...
    SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones);
    //      0      n      1    n+1      2    n+2      3    n+3 ...
    Idx =
        DAG.getNode(RISCVISD::ADD_VL, DL, IdxVT, Idx, VLMax, Idx, OddMask, VL);
 
    // Then perform the interleave
    //   v[0]   v[n]   v[1] v[n+1]   v[2] v[n+2]   v[3] v[n+3] ...
    SDValue TrueMask = getAllOnesMask(IdxVT, VL, DL, DAG);
    Interleaved = DAG.getNode(RISCVISD::VRGATHEREI16_VV_VL, DL, ConcatVT,
                              Concat, Idx, DAG.getUNDEF(ConcatVT), TrueMask, VL);
  }
 
  // Extract the two halves from the interleaved result
  SDValue Lo = DAG.getExtractSubvector(DL, VecVT, Interleaved, 0);
  SDValue Hi = DAG.getExtractSubvector(DL, VecVT, Interleaved,
                                       VecVT.getVectorMinNumElements());
 
  return DAG.getMergeValues({Lo, Hi}, DL);
}
 
// Lower step_vector to the vid instruction. Any non-identity step value must
// be accounted for my manual expansion.
SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  assert(VT.isScalableVector() && "Expected scalable vector");
  MVT XLenVT = Subtarget.getXLenVT();
  auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
  SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
  uint64_t StepValImm = Op.getConstantOperandVal(0);
  if (StepValImm != 1) {
    if (isPowerOf2_64(StepValImm)) {
      SDValue StepVal =
          DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
                      DAG.getConstant(Log2_64(StepValImm), DL, XLenVT), VL);
      StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
    } else {
      SDValue StepVal = lowerScalarSplat(
          SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()),
          VL, VT, DL, DAG, Subtarget);
      StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
    }
  }
  return StepVec;
}
 
// Implement vector_reverse using vrgather.vv with indices determined by
// subtracting the id of each element from (VLMAX-1). This will convert
// the indices like so:
// (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
// TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VecVT = Op.getSimpleValueType();
 
  // Reverse a 64-bit packed vector on RV32 by reversing each 32-bit half and
  // swapping them.
  if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
    assert(!Subtarget.is64Bit() && VecVT.getSizeInBits() == 64 &&
           "Unexpected packed VECTOR_REVERSE type");
    SDValue V = Op.getOperand(0);
    if (VecVT == MVT::v2i32) {
      // A 2-element reverse is just an element swap.
      SDValue Lo = DAG.getExtractVectorElt(DL, MVT::i32, V, 0);
      SDValue Hi = DAG.getExtractVectorElt(DL, MVT::i32, V, 1);
      return DAG.getBuildVector(VecVT, DL, {Hi, Lo});
    }
    auto [Lo, Hi] = DAG.SplitVector(V, DL);
    Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, Lo.getSimpleValueType(), Lo);
    Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, Hi.getSimpleValueType(), Hi);
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Hi, Lo);
  }
 
  if (VecVT.getVectorElementType() == MVT::i1) {
    MVT WidenVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
    SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0));
    SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
    return DAG.getSetCC(DL, VecVT, Op2,
                        DAG.getConstant(0, DL, Op2.getValueType()), ISD::SETNE);
  }
 
  MVT ContainerVT = VecVT;
  SDValue Vec = Op.getOperand(0);
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
 
  MVT XLenVT = Subtarget.getXLenVT();
  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
  // On some uarchs vrgather.vv will read from every input register for each
  // output register, regardless of the indices. However to reverse a vector
  // each output register only needs to read from one register. So decompose it
  // into LMUL * M1 vrgather.vvs, so we get O(LMUL) performance instead of
  // O(LMUL^2).
  //
  // vsetvli a1, zero, e64, m4, ta, ma
  // vrgatherei16.vv v12, v8, v16
  // ->
  // vsetvli a1, zero, e64, m1, ta, ma
  // vrgather.vv v15, v8, v16
  // vrgather.vv v14, v9, v16
  // vrgather.vv v13, v10, v16
  // vrgather.vv v12, v11, v16
  if (ContainerVT.bitsGT(RISCVTargetLowering::getM1VT(ContainerVT)) &&
      ContainerVT.getVectorElementCount().isKnownMultipleOf(2)) {
    auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
    Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, Lo.getValueType(), Lo);
    Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, Hi.getValueType(), Hi);
    SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ContainerVT, Hi, Lo);
 
    // Fixed length vectors might not fit exactly into their container, and so
    // leave a gap in the front of the vector after being reversed. Slide this
    // away.
    //
    // x x x x 3 2 1 0 <- v4i16 @ vlen=128
    // 0 1 2 3 x x x x <- reverse
    // x x x x 0 1 2 3 <- vslidedown.vx
    if (VecVT.isFixedLengthVector()) {
      SDValue Offset = DAG.getNode(
          ISD::SUB, DL, XLenVT,
          DAG.getElementCount(DL, XLenVT, ContainerVT.getVectorElementCount()),
          DAG.getElementCount(DL, XLenVT, VecVT.getVectorElementCount()));
      Concat =
          getVSlidedown(DAG, Subtarget, DL, ContainerVT,
                        DAG.getUNDEF(ContainerVT), Concat, Offset, Mask, VL);
      Concat = convertFromScalableVector(VecVT, Concat, DAG, Subtarget);
    }
    return Concat;
  }
 
  unsigned EltSize = ContainerVT.getScalarSizeInBits();
  unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
  unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
  unsigned MaxVLMAX =
      VecVT.isFixedLengthVector()
          ? VecVT.getVectorNumElements()
          : RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
 
  unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
 
  // If this is SEW=8 and VLMAX is potentially more than 256, we need
  // to use vrgatherei16.vv.
  if (MaxVLMAX > 256 && EltSize == 8) {
    // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
    // Reverse each half, then reassemble them in reverse order.
    // NOTE: It's also possible that after splitting that VLMAX no longer
    // requires vrgatherei16.vv.
    if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
      auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
      auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
      Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
      Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
      // Reassemble the low and high pieces reversed.
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Hi, Lo);
    }
 
    // Just promote the int type to i16 which will double the LMUL.
    IntVT = MVT::getVectorVT(MVT::i16, ContainerVT.getVectorElementCount());
    GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
  }
 
  // At LMUL > 1, do the index computation in 16 bits to reduce register
  // pressure.
  if (IntVT.getScalarType().bitsGT(MVT::i16) &&
      IntVT.bitsGT(RISCVTargetLowering::getM1VT(IntVT))) {
    assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
    GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
    IntVT = IntVT.changeVectorElementType(MVT::i16);
  }
 
  // Calculate VLMAX-1 for the desired SEW.
  SDValue VLMinus1 = DAG.getNode(
      ISD::SUB, DL, XLenVT,
      DAG.getElementCount(DL, XLenVT, VecVT.getVectorElementCount()),
      DAG.getConstant(1, DL, XLenVT));
 
  // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
  bool IsRV32E64 =
      !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
  SDValue SplatVL;
  if (!IsRV32E64)
    SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
  else
    SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, DAG.getUNDEF(IntVT),
                          VLMinus1, DAG.getRegister(RISCV::X0, XLenVT));
 
  SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
  SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
                                DAG.getUNDEF(IntVT), Mask, VL);
 
  SDValue Gather = DAG.getNode(GatherOpc, DL, ContainerVT, Vec, Indices,
                               DAG.getUNDEF(ContainerVT), Mask, VL);
  if (VecVT.isFixedLengthVector())
    Gather = convertFromScalableVector(VecVT, Gather, DAG, Subtarget);
  return Gather;
}
 
SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue V1 = Op.getOperand(0);
  SDValue V2 = Op.getOperand(1);
  SDValue Offset = Op.getOperand(2);
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VecVT = Op.getSimpleValueType();
 
  SDValue VLMax = computeVLMax(VecVT, DL, DAG);
 
  SDValue DownOffset, UpOffset;
  if (Op.getOpcode() == ISD::VECTOR_SPLICE_LEFT) {
    // The operand is a TargetConstant, we need to rebuild it as a regular
    // constant.
    DownOffset = Offset;
    UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, Offset);
  } else {
    // The operand is a TargetConstant, we need to rebuild it as a regular
    // constant rather than negating the original operand.
    UpOffset = Offset;
    DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, Offset);
  }
 
  SDValue TrueMask = getAllOnesMask(VecVT, VLMax, DL, DAG);
 
  SDValue SlideDown = getVSlidedown(
      DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1, DownOffset, TrueMask,
      Subtarget.hasVLDependentLatency() ? UpOffset
                                        : DAG.getRegister(RISCV::X0, XLenVT));
  return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset,
                     TrueMask, DAG.getRegister(RISCV::X0, XLenVT),
                     RISCVVType::TAIL_AGNOSTIC);
}
 
SDValue
RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
                                                     SelectionDAG &DAG) const {
  SDLoc DL(Op);
  auto *Load = cast<LoadSDNode>(Op);
 
  assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                        Load->getMemoryVT(),
                                        *Load->getMemOperand()) &&
         "Expecting a correctly-aligned load");
 
  MVT VT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
  MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
  // If we know the exact VLEN and our fixed length vector completely fills
  // the container, use a whole register load instead.
  const auto [MinVLMAX, MaxVLMAX] =
      RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
  if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
      RISCVTargetLowering::getM1VT(ContainerVT).bitsLE(ContainerVT)) {
    MachineMemOperand *MMO = Load->getMemOperand();
    SDValue NewLoad =
        DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
                    MMO->getPointerInfo(), MMO->getBaseAlign(), MMO->getFlags(),
                    MMOMetadata(MMO->getAAInfo(), MMO->getRanges()));
    SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
    return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
  }
 
  SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
 
  bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
  SDValue IntID = DAG.getTargetConstant(
      IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, XLenVT);
  SmallVector<SDValue, 4> Ops{Load->getChain(), IntID};
  if (!IsMaskOp)
    Ops.push_back(DAG.getUNDEF(ContainerVT));
  Ops.push_back(Load->getBasePtr());
  Ops.push_back(VL);
  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
  SDValue NewLoad =
      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
                              Load->getMemoryVT(), Load->getMemOperand());
 
  SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
  return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
}
 
SDValue
RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
                                                      SelectionDAG &DAG) const {
  SDLoc DL(Op);
  auto *Store = cast<StoreSDNode>(Op);
 
  assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                        Store->getMemoryVT(),
                                        *Store->getMemOperand()) &&
         "Expecting a correctly-aligned store");
 
  SDValue StoreVal = Store->getValue();
  MVT VT = StoreVal.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  // If the size less than a byte, we need to pad with zeros to make a byte.
  if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
    VT = MVT::v8i1;
    StoreVal =
        DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, VT), StoreVal, 0);
  }
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
  SDValue NewValue =
      convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
 
  // If we know the exact VLEN and our fixed length vector completely fills
  // the container, use a whole register store instead.
  const auto [MinVLMAX, MaxVLMAX] =
      RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
  if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
      RISCVTargetLowering::getM1VT(ContainerVT).bitsLE(ContainerVT)) {
    MachineMemOperand *MMO = Store->getMemOperand();
    return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
                        MMO->getPointerInfo(), MMO->getBaseAlign(),
                        MMO->getFlags(), MMO->getAAInfo());
  }
 
  SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
 
  bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
  SDValue IntID = DAG.getTargetConstant(
      IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, XLenVT);
  return DAG.getMemIntrinsicNode(
      ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
      {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
      Store->getMemoryVT(), Store->getMemOperand());
}
 
SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
                                             SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
 
  const auto *MemSD = cast<MemSDNode>(Op);
  EVT MemVT = MemSD->getMemoryVT();
  MachineMemOperand *MMO = MemSD->getMemOperand();
  SDValue Chain = MemSD->getChain();
  SDValue BasePtr = MemSD->getBasePtr();
 
  SDValue Mask, PassThru, VL;
  bool IsExpandingLoad = false;
  if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
    Mask = VPLoad->getMask();
    PassThru = DAG.getUNDEF(VT);
    VL = VPLoad->getVectorLength();
  } else {
    const auto *MLoad = cast<MaskedLoadSDNode>(Op);
    Mask = MLoad->getMask();
    PassThru = MLoad->getPassThru();
    IsExpandingLoad = MLoad->isExpandingLoad();
  }
 
  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
 
  MVT XLenVT = Subtarget.getXLenVT();
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
    if (!IsUnmasked) {
      MVT MaskVT = getMaskTypeFor(ContainerVT);
      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
    }
  }
 
  if (!VL)
    VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
 
  SDValue ExpandingVL;
  if (!IsUnmasked && IsExpandingLoad) {
    ExpandingVL = VL;
    VL =
        DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
                    getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL);
  }
 
  unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
                                                 : Intrinsic::riscv_vle_mask;
  SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
  if (IntID == Intrinsic::riscv_vle)
    Ops.push_back(DAG.getUNDEF(ContainerVT));
  else
    Ops.push_back(PassThru);
  Ops.push_back(BasePtr);
  if (IntID == Intrinsic::riscv_vle_mask)
    Ops.push_back(Mask);
  Ops.push_back(VL);
  if (IntID == Intrinsic::riscv_vle_mask)
    Ops.push_back(DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT));
 
  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
 
  SDValue Result =
      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
  Chain = Result.getValue(1);
  if (ExpandingVL) {
    MVT IndexVT = ContainerVT;
    if (ContainerVT.isFloatingPoint())
      IndexVT = ContainerVT.changeVectorElementTypeToInteger();
 
    MVT IndexEltVT = IndexVT.getVectorElementType();
    bool UseVRGATHEREI16 = false;
    // If index vector is an i8 vector and the element count exceeds 256, we
    // should change the element type of index vector to i16 to avoid
    // overflow.
    uint64_t MaxEltCount = VT.getVectorMinNumElements();
    if (VT.isScalableVector())
      MaxEltCount *= Subtarget.getRealMaxVLen() / RISCV::RVVBitsPerBlock;
    if (IndexEltVT == MVT::i8 && MaxEltCount > 256) {
      // FIXME: We need to do vector splitting manually for LMUL=8 cases.
      assert(getLMUL(IndexVT) != RISCVVType::LMUL_8);
      IndexVT = IndexVT.changeVectorElementType(MVT::i16);
      UseVRGATHEREI16 = true;
    }
 
    SDValue Iota =
        DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
                    DAG.getTargetConstant(Intrinsic::riscv_viota, DL, XLenVT),
                    DAG.getUNDEF(IndexVT), Mask, ExpandingVL);
    Result =
        DAG.getNode(UseVRGATHEREI16 ? RISCVISD::VRGATHEREI16_VV_VL
                                    : RISCVISD::VRGATHER_VV_VL,
                    DL, ContainerVT, Result, Iota, PassThru, Mask, ExpandingVL);
  }
 
  if (VT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
 
  return DAG.getMergeValues({Result, Chain}, DL);
}
 
SDValue RISCVTargetLowering::lowerLoadFF(SDValue Op, SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op->getSimpleValueType(0);
 
  const auto *VPLoadFF = cast<VPLoadFFSDNode>(Op);
  EVT MemVT = VPLoadFF->getMemoryVT();
  MachineMemOperand *MMO = VPLoadFF->getMemOperand();
  SDValue Chain = VPLoadFF->getChain();
  SDValue BasePtr = VPLoadFF->getBasePtr();
 
  SDValue Mask = VPLoadFF->getMask();
  SDValue VL = VPLoadFF->getVectorLength();
 
  MVT XLenVT = Subtarget.getXLenVT();
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    MVT MaskVT = getMaskTypeFor(ContainerVT);
    Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
  }
 
  unsigned IntID = Intrinsic::riscv_vleff_mask;
  SDValue Ops[] = {
      Chain,
      DAG.getTargetConstant(IntID, DL, XLenVT),
      DAG.getUNDEF(ContainerVT),
      BasePtr,
      Mask,
      VL,
      DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT)};
 
  SDVTList VTs = DAG.getVTList({ContainerVT, Op->getValueType(1), MVT::Other});
 
  SDValue Result =
      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
  SDValue OutVL = Result.getValue(1);
  Chain = Result.getValue(2);
 
  if (VT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
 
  return DAG.getMergeValues({Result, OutVL, Chain}, DL);
}
 
SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
 
  const auto *MemSD = cast<MemSDNode>(Op);
  EVT MemVT = MemSD->getMemoryVT();
  MachineMemOperand *MMO = MemSD->getMemOperand();
  SDValue Chain = MemSD->getChain();
  SDValue BasePtr = MemSD->getBasePtr();
  SDValue Val, Mask, VL;
 
  bool IsCompressingStore = false;
  if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
    Val = VPStore->getValue();
    Mask = VPStore->getMask();
    VL = VPStore->getVectorLength();
  } else {
    const auto *MStore = cast<MaskedStoreSDNode>(Op);
    Val = MStore->getValue();
    Mask = MStore->getMask();
    IsCompressingStore = MStore->isCompressingStore();
  }
 
  bool IsUnmasked =
      ISD::isConstantSplatVectorAllOnes(Mask.getNode()) || IsCompressingStore;
 
  MVT VT = Val.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
 
    Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
    if (!IsUnmasked || IsCompressingStore) {
      MVT MaskVT = getMaskTypeFor(ContainerVT);
      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
    }
  }
 
  if (!VL)
    VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
 
  if (IsCompressingStore) {
    Val = DAG.getNode(
        ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
        DAG.getTargetConstant(Intrinsic::riscv_vcompress, DL, XLenVT),
        DAG.getUNDEF(ContainerVT), Val, Mask, VL);
    VL =
        DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
                    getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL);
  }
 
  unsigned IntID =
      IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
  SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
  Ops.push_back(Val);
  Ops.push_back(BasePtr);
  if (!IsUnmasked)
    Ops.push_back(Mask);
  Ops.push_back(VL);
 
  return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
                                 DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
}
 
SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Val = Op.getOperand(0);
  SDValue Mask = Op.getOperand(1);
  SDValue Passthru = Op.getOperand(2);
 
  MVT VT = Val.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    MVT MaskVT = getMaskTypeFor(ContainerVT);
    Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
    Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
    Passthru = convertToScalableVector(ContainerVT, Passthru, DAG, Subtarget);
  }
 
  SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
  SDValue Res =
      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
                  DAG.getTargetConstant(Intrinsic::riscv_vcompress, DL, XLenVT),
                  Passthru, Val, Mask, VL);
 
  if (VT.isFixedLengthVector())
    Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
 
  return Res;
}
 
SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op,
                                                     SelectionDAG &DAG) const {
  unsigned Opc = Op.getOpcode();
  SDLoc DL(Op);
  SDValue Chain = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  SDValue Op2 = Op.getOperand(2);
  SDValue CC = Op.getOperand(3);
  ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
  MVT VT = Op.getSimpleValueType();
  MVT InVT = Op1.getSimpleValueType();
 
  // RVV VMFEQ/VMFNE ignores qNan, so we expand strict_fsetccs with OEQ/UNE
  // condition code.
  if (Opc == ISD::STRICT_FSETCCS) {
    // Expand strict_fsetccs(x, oeq) to
    // (and strict_fsetccs(x, y, oge), strict_fsetccs(x, y, ole))
    SDVTList VTList = Op->getVTList();
    if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) {
      SDValue OLECCVal = DAG.getCondCode(ISD::SETOLE);
      SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
                                 Op2, OLECCVal);
      SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2,
                                 Op1, OLECCVal);
      SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                                     Tmp1.getValue(1), Tmp2.getValue(1));
      // Tmp1 and Tmp2 might be the same node.
      if (Tmp1 != Tmp2)
        Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
      return DAG.getMergeValues({Tmp1, OutChain}, DL);
    }
 
    // Expand (strict_fsetccs x, y, une) to (not (strict_fsetccs x, y, oeq))
    if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) {
      SDValue OEQCCVal = DAG.getCondCode(ISD::SETOEQ);
      SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
                                Op2, OEQCCVal);
      SDValue Res = DAG.getNOT(DL, OEQ, VT);
      return DAG.getMergeValues({Res, OEQ.getValue(1)}, DL);
    }
  }
 
  MVT ContainerInVT = InVT;
  if (InVT.isFixedLengthVector()) {
    ContainerInVT = getContainerForFixedLengthVector(InVT);
    Op1 = convertToScalableVector(ContainerInVT, Op1, DAG, Subtarget);
    Op2 = convertToScalableVector(ContainerInVT, Op2, DAG, Subtarget);
  }
  MVT MaskVT = getMaskTypeFor(ContainerInVT);
 
  auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget);
 
  SDValue Res;
  if (Opc == ISD::STRICT_FSETCC &&
      (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE ||
       CCVal == ISD::SETOLE)) {
    // VMFLT/VMFLE/VMFGT/VMFGE raise exception for qNan. Generate a mask to only
    // active when both input elements are ordered.
    SDValue True = getAllOnesMask(ContainerInVT, VL, DL, DAG);
    SDValue OrderMask1 = DAG.getNode(
        RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
        {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
         True, VL});
    SDValue OrderMask2 = DAG.getNode(
        RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
        {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
         True, VL});
    Mask =
        DAG.getNode(RISCVISD::VMAND_VL, DL, MaskVT, OrderMask1, OrderMask2, VL);
    // Use Mask as the passthru operand to let the result be 0 if either of the
    // inputs is unordered.
    Res = DAG.getNode(RISCVISD::STRICT_FSETCCS_VL, DL,
                      DAG.getVTList(MaskVT, MVT::Other),
                      {Chain, Op1, Op2, CC, Mask, Mask, VL});
  } else {
    unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
                                                : RISCVISD::STRICT_FSETCCS_VL;
    Res = DAG.getNode(RVVOpc, DL, DAG.getVTList(MaskVT, MVT::Other),
                      {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
  }
 
  if (VT.isFixedLengthVector()) {
    SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
    return DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
  }
  return Res;
}
 
// Lower vector ABS to smax(X, sub(0, X)).
SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue X = Op.getOperand(0);
 
  assert(VT.isFixedLengthVector() && "Unexpected type for ISD::ABS");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT);
  X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  SDValue SplatZero = DAG.getNode(
      RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
      DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
  SDValue Result;
  if (Subtarget.hasStdExtZvabd()) {
    Result = DAG.getNode(RISCVISD::ABDS_VL, DL, ContainerVT, X, SplatZero,
                         DAG.getUNDEF(ContainerVT), Mask, VL);
  } else {
    SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
                               DAG.getUNDEF(ContainerVT), Mask, VL);
    Result = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
                         DAG.getUNDEF(ContainerVT), Mask, VL);
  }
  return convertFromScalableVector(VT, Result, DAG, Subtarget);
}
 
SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op,
                                               SelectionDAG &DAG) const {
  const auto &TSInfo =
      static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
 
  unsigned NewOpc = getRISCVVLOp(Op);
  bool HasPassthruOp = TSInfo.hasPassthruOp(NewOpc);
  bool HasMask = TSInfo.hasMaskOp(NewOpc);
 
  MVT VT = Op.getSimpleValueType();
  MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
  // Create list of operands by converting existing ones to scalable types.
  SmallVector<SDValue, 6> Ops;
  for (const SDValue &V : Op->op_values()) {
    assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
 
    // Pass through non-vector operands.
    if (!V.getValueType().isVector()) {
      Ops.push_back(V);
      continue;
    }
 
    // "cast" fixed length vector to a scalable vector.
    assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
           "Only fixed length vectors are supported!");
    MVT VContainerVT = ContainerVT.changeVectorElementType(
        V.getSimpleValueType().getVectorElementType());
    Ops.push_back(convertToScalableVector(VContainerVT, V, DAG, Subtarget));
  }
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
  if (HasPassthruOp)
    Ops.push_back(DAG.getUNDEF(ContainerVT));
  if (HasMask)
    Ops.push_back(Mask);
  Ops.push_back(VL);
 
  // StrictFP operations have two result values. Their lowered result should
  // have same result count.
  if (Op->isStrictFPOpcode()) {
    SDValue ScalableRes =
        DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops,
                    Op->getFlags());
    SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
    return DAG.getMergeValues({SubVec, ScalableRes.getValue(1)}, DL);
  }
 
  SDValue ScalableRes =
      DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags());
  return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
}
 
// Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
// * Operands of each node are assumed to be in the same order.
// * The EVL operand is promoted from i32 to i64 on RV64.
// * Fixed-length vectors are converted to their scalable-vector container
//   types.
SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
  const auto &TSInfo =
      static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
 
  unsigned RISCVISDOpc = getRISCVVLOp(Op);
  bool HasPassthruOp = TSInfo.hasPassthruOp(RISCVISDOpc);
 
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SmallVector<SDValue, 4> Ops;
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VT);
 
  for (const auto &OpIdx : enumerate(Op->ops())) {
    SDValue V = OpIdx.value();
    assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
    // Add dummy passthru value before the mask. Or if there isn't a mask,
    // before EVL.
    if (HasPassthruOp) {
      auto MaskIdx = ISD::getVPMaskIdx(Op.getOpcode());
      if (MaskIdx) {
        if (*MaskIdx == OpIdx.index())
          Ops.push_back(DAG.getUNDEF(ContainerVT));
      } else if (ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) ==
                 OpIdx.index()) {
        assert(Op.getOpcode() == ISD::VP_MERGE);
        // For VP_MERGE, copy the false operand instead of an undef value.
        Ops.push_back(Ops.back());
      }
    }
    // VFCVT_RM_X_F_VL requires a rounding mode to be injected before the VL.
    if (RISCVISDOpc == RISCVISD::VFCVT_RM_X_F_VL &&
        ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) == OpIdx.index())
      Ops.push_back(DAG.getTargetConstant(RISCVFPRndMode::DYN, DL,
                                          Subtarget.getXLenVT()));
    // Pass through operands which aren't fixed-length vectors.
    if (!V.getValueType().isFixedLengthVector()) {
      Ops.push_back(V);
      continue;
    }
    // "cast" fixed length vector to a scalable vector.
    MVT OpVT = V.getSimpleValueType();
    MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
    assert(useRVVForFixedLengthVectorVT(OpVT) &&
           "Only fixed length vectors are supported!");
    Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
  }
 
  if (!VT.isFixedLengthVector())
    return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
 
  SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
 
  return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
}
 
SDValue RISCVTargetLowering::lowerVPMergeMask(SDValue Op,
                                              SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  SDValue Mask = Op.getOperand(0);
  SDValue TrueVal = Op.getOperand(1);
  SDValue FalseVal = Op.getOperand(2);
  SDValue VL = Op.getOperand(3);
 
  // Use default legalization if a vector of EVL type would be legal.
  EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), VL.getValueType(),
                                  VT.getVectorElementCount());
  if (isTypeLegal(EVLVecVT))
    return SDValue();
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    Mask = convertToScalableVector(ContainerVT, Mask, DAG, Subtarget);
    TrueVal = convertToScalableVector(ContainerVT, TrueVal, DAG, Subtarget);
    FalseVal = convertToScalableVector(ContainerVT, FalseVal, DAG, Subtarget);
  }
 
  // Promote to a vector of i8.
  MVT PromotedVT = ContainerVT.changeVectorElementType(MVT::i8);
 
  // Promote TrueVal and FalseVal using VLMax.
  // FIXME: Is there a better way to do this?
  SDValue VLMax = DAG.getRegister(RISCV::X0, XLenVT);
  SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, PromotedVT,
                                 DAG.getUNDEF(PromotedVT),
                                 DAG.getConstant(1, DL, XLenVT), VLMax);
  SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, PromotedVT,
                                  DAG.getUNDEF(PromotedVT),
                                  DAG.getConstant(0, DL, XLenVT), VLMax);
  TrueVal = DAG.getNode(RISCVISD::VMERGE_VL, DL, PromotedVT, TrueVal, SplatOne,
                        SplatZero, DAG.getUNDEF(PromotedVT), VL);
  // Any element past VL uses FalseVal, so use VLMax
  FalseVal = DAG.getNode(RISCVISD::VMERGE_VL, DL, PromotedVT, FalseVal,
                         SplatOne, SplatZero, DAG.getUNDEF(PromotedVT), VLMax);
 
  // VP_MERGE the two promoted values.
  SDValue VPMerge = DAG.getNode(RISCVISD::VMERGE_VL, DL, PromotedVT, Mask,
                                TrueVal, FalseVal, FalseVal, VL);
 
  // Convert back to mask.
  SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
  SDValue Result = DAG.getNode(
      RISCVISD::SETCC_VL, DL, ContainerVT,
      {VPMerge, DAG.getConstant(0, DL, PromotedVT), DAG.getCondCode(ISD::SETNE),
       DAG.getUNDEF(getMaskTypeFor(ContainerVT)), TrueMask, VLMax});
 
  if (VT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
  return Result;
}
 
SDValue
RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
                                               SelectionDAG &DAG) const {
  using namespace SDPatternMatch;
 
  SDLoc DL(Op);
 
  SDValue Op1 = Op.getOperand(0);
  SDValue Op2 = Op.getOperand(1);
  SDValue Offset = Op.getOperand(2);
  SDValue Mask = Op.getOperand(3);
  SDValue EVL1 = Op.getOperand(4);
  SDValue EVL2 = Op.getOperand(5);
 
  const MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
    Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
    MVT MaskVT = getMaskTypeFor(ContainerVT);
    Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
  }
 
  bool IsMaskVector = VT.getVectorElementType() == MVT::i1;
  if (IsMaskVector) {
    ContainerVT = ContainerVT.changeVectorElementType(MVT::i8);
 
    // Expand input operands
    SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                      DAG.getUNDEF(ContainerVT),
                                      DAG.getConstant(1, DL, XLenVT), EVL1);
    SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                       DAG.getUNDEF(ContainerVT),
                                       DAG.getConstant(0, DL, XLenVT), EVL1);
    Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1,
                      SplatZeroOp1, DAG.getUNDEF(ContainerVT), EVL1);
 
    SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                      DAG.getUNDEF(ContainerVT),
                                      DAG.getConstant(1, DL, XLenVT), EVL2);
    SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                       DAG.getUNDEF(ContainerVT),
                                       DAG.getConstant(0, DL, XLenVT), EVL2);
    Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2,
                      SplatZeroOp2, DAG.getUNDEF(ContainerVT), EVL2);
  }
 
  auto getVectorFirstEle = [](SDValue Vec) {
    SDValue FirstEle;
    if (sd_match(Vec, m_InsertElt(m_Value(), m_Value(FirstEle), m_Zero())))
      return FirstEle;
 
    if (Vec.getOpcode() == ISD::SPLAT_VECTOR ||
        Vec.getOpcode() == ISD::BUILD_VECTOR)
      return Vec.getOperand(0);
 
    return SDValue();
  };
 
  if (!IsMaskVector && isNullConstant(Offset) && isOneConstant(EVL1))
    if (auto FirstEle = getVectorFirstEle(Op->getOperand(0))) {
      MVT EltVT = ContainerVT.getVectorElementType();
      SDValue Result;
      if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
          (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
        EltVT = EltVT.changeTypeToInteger();
        ContainerVT = ContainerVT.changeVectorElementType(EltVT);
        Op2 = DAG.getBitcast(ContainerVT, Op2);
        FirstEle =
            DAG.getAnyExtOrTrunc(DAG.getBitcast(EltVT, FirstEle), DL, XLenVT);
      }
      Result = DAG.getNode(EltVT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL
                                                   : RISCVISD::VSLIDE1UP_VL,
                           DL, ContainerVT, DAG.getUNDEF(ContainerVT), Op2,
                           FirstEle, Mask, EVL2);
      Result = DAG.getBitcast(
          ContainerVT.changeVectorElementType(VT.getVectorElementType()),
          Result);
      return VT.isFixedLengthVector()
                 ? convertFromScalableVector(VT, Result, DAG, Subtarget)
                 : Result;
    }
 
  int64_t ImmValue = cast<ConstantSDNode>(Offset)->getSExtValue();
  SDValue DownOffset, UpOffset;
  if (ImmValue >= 0) {
    // The operand is a TargetConstant, we need to rebuild it as a regular
    // constant.
    DownOffset = DAG.getConstant(ImmValue, DL, XLenVT);
    UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset);
  } else {
    // The operand is a TargetConstant, we need to rebuild it as a regular
    // constant rather than negating the original operand.
    UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
    DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset);
  }
 
  if (ImmValue != 0)
    Op1 = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
                        DAG.getUNDEF(ContainerVT), Op1, DownOffset, Mask,
                        Subtarget.hasVLDependentLatency() ? UpOffset : EVL2);
  SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, Op1, Op2,
                               UpOffset, Mask, EVL2, RISCVVType::TAIL_AGNOSTIC);
 
  if (IsMaskVector) {
    // Truncate Result back to a mask vector (Result has same EVL as Op2)
    Result = DAG.getNode(
        RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1),
        {Result, DAG.getConstant(0, DL, ContainerVT),
         DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
         Mask, EVL2});
  }
 
  if (!VT.isFixedLengthVector())
    return Result;
  return convertFromScalableVector(VT, Result, DAG, Subtarget);
}
 
SDValue
RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  SDValue Op1 = Op.getOperand(0);
  SDValue Mask = Op.getOperand(1);
  SDValue EVL = Op.getOperand(2);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
    MVT MaskVT = getMaskTypeFor(ContainerVT);
    Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
  }
 
  MVT GatherVT = ContainerVT;
  MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger();
  // Check if we are working with mask vectors
  bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1;
  if (IsMaskVector) {
    GatherVT = IndicesVT = ContainerVT.changeVectorElementType(MVT::i8);
 
    // Expand input operand
    SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
                                   DAG.getUNDEF(IndicesVT),
                                   DAG.getConstant(1, DL, XLenVT), EVL);
    SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
                                    DAG.getUNDEF(IndicesVT),
                                    DAG.getConstant(0, DL, XLenVT), EVL);
    Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, IndicesVT, Op1, SplatOne,
                      SplatZero, DAG.getUNDEF(IndicesVT), EVL);
  }
 
  unsigned EltSize = GatherVT.getScalarSizeInBits();
  unsigned MinSize = GatherVT.getSizeInBits().getKnownMinValue();
  unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
  unsigned MaxVLMAX =
      RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
 
  unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
  // If this is SEW=8 and VLMAX is unknown or more than 256, we need
  // to use vrgatherei16.vv.
  // TODO: It's also possible to use vrgatherei16.vv for other types to
  // decrease register width for the index calculation.
  // NOTE: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
  if (MaxVLMAX > 256 && EltSize == 8) {
    // If this is LMUL=8, we have to split before using vrgatherei16.vv.
    // Split the vector in half and reverse each half using a full register
    // reverse.
    // Swap the halves and concatenate them.
    // Slide the concatenated result by (VLMax - VL).
    if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
      auto [LoVT, HiVT] = DAG.GetSplitDestVTs(GatherVT);
      auto [Lo, Hi] = DAG.SplitVector(Op1, DL);
 
      SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
      SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
 
      // Reassemble the low and high pieces reversed.
      // NOTE: this Result is unmasked (because we do not need masks for
      // shuffles). If in the future this has to change, we can use a SELECT_VL
      // between Result and UNDEF using the mask originally passed to VP_REVERSE
      SDValue Result =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev);
 
      // Slide off any elements from past EVL that were reversed into the low
      // elements.
      SDValue VLMax =
          DAG.getElementCount(DL, XLenVT, GatherVT.getVectorElementCount());
      SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL);
 
      Result = getVSlidedown(DAG, Subtarget, DL, GatherVT,
                             DAG.getUNDEF(GatherVT), Result, Diff, Mask, EVL);
 
      if (IsMaskVector) {
        // Truncate Result back to a mask vector
        Result =
            DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
                        {Result, DAG.getConstant(0, DL, GatherVT),
                         DAG.getCondCode(ISD::SETNE),
                         DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
      }
 
      if (!VT.isFixedLengthVector())
        return Result;
      return convertFromScalableVector(VT, Result, DAG, Subtarget);
    }
 
    // Just promote the int type to i16 which will double the LMUL.
    IndicesVT = MVT::getVectorVT(MVT::i16, IndicesVT.getVectorElementCount());
    GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
  }
 
  SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IndicesVT, Mask, EVL);
  SDValue VecLen =
      DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT));
  SDValue VecLenSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
                                    DAG.getUNDEF(IndicesVT), VecLen, EVL);
  SDValue VRSUB = DAG.getNode(RISCVISD::SUB_VL, DL, IndicesVT, VecLenSplat, VID,
                              DAG.getUNDEF(IndicesVT), Mask, EVL);
  SDValue Result = DAG.getNode(GatherOpc, DL, GatherVT, Op1, VRSUB,
                               DAG.getUNDEF(GatherVT), Mask, EVL);
 
  if (IsMaskVector) {
    // Truncate Result back to a mask vector
    Result = DAG.getNode(
        RISCVISD::SETCC_VL, DL, ContainerVT,
        {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE),
         DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
  }
 
  if (!VT.isFixedLengthVector())
    return Result;
  return convertFromScalableVector(VT, Result, DAG, Subtarget);
}
 
SDValue RISCVTargetLowering::lowerVPStridedLoad(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VT);
 
  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
 
  auto *VPNode = cast<VPStridedLoadSDNode>(Op);
  // Check if the mask is known to be all ones
  SDValue Mask = VPNode->getMask();
  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
 
  SDValue IntID = DAG.getTargetConstant(IsUnmasked ? Intrinsic::riscv_vlse
                                                   : Intrinsic::riscv_vlse_mask,
                                        DL, XLenVT);
  SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID,
                              DAG.getUNDEF(ContainerVT), VPNode->getBasePtr(),
                              VPNode->getStride()};
  if (!IsUnmasked) {
    if (VT.isFixedLengthVector()) {
      MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1);
      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
    }
    Ops.push_back(Mask);
  }
  Ops.push_back(VPNode->getVectorLength());
  if (!IsUnmasked) {
    SDValue Policy =
        DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT);
    Ops.push_back(Policy);
  }
 
  SDValue Result =
      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
                              VPNode->getMemoryVT(), VPNode->getMemOperand());
  SDValue Chain = Result.getValue(1);
 
  if (VT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
 
  return DAG.getMergeValues({Result, Chain}, DL);
}
 
SDValue RISCVTargetLowering::lowerVPStridedStore(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
 
  auto *VPNode = cast<VPStridedStoreSDNode>(Op);
  SDValue StoreVal = VPNode->getValue();
  MVT VT = StoreVal.getSimpleValueType();
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
  }
 
  // Check if the mask is known to be all ones
  SDValue Mask = VPNode->getMask();
  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
 
  SDValue IntID = DAG.getTargetConstant(IsUnmasked ? Intrinsic::riscv_vsse
                                                   : Intrinsic::riscv_vsse_mask,
                                        DL, XLenVT);
  SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID, StoreVal,
                              VPNode->getBasePtr(), VPNode->getStride()};
  if (!IsUnmasked) {
    if (VT.isFixedLengthVector()) {
      MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1);
      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
    }
    Ops.push_back(Mask);
  }
  Ops.push_back(VPNode->getVectorLength());
 
  return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(),
                                 Ops, VPNode->getMemoryVT(),
                                 VPNode->getMemOperand());
}
 
// Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
// matched to a RVV indexed load. The RVV indexed load instructions only
// support the "unsigned unscaled" addressing mode; indices are implicitly
// zero-extended or truncated to XLEN and are treated as byte offsets. Any
// signed or scaled indexing is extended to the XLEN value type and scaled
// accordingly.
SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
                                               SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
 
  const auto *MemSD = cast<MemSDNode>(Op.getNode());
  EVT MemVT = MemSD->getMemoryVT();
  MachineMemOperand *MMO = MemSD->getMemOperand();
  SDValue Chain = MemSD->getChain();
  SDValue BasePtr = MemSD->getBasePtr();
 
  [[maybe_unused]] ISD::LoadExtType LoadExtType;
  SDValue Index, Mask, PassThru, VL;
 
  if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
    Index = VPGN->getIndex();
    Mask = VPGN->getMask();
    PassThru = DAG.getUNDEF(VT);
    VL = VPGN->getVectorLength();
    // VP doesn't support extending loads.
    LoadExtType = ISD::NON_EXTLOAD;
  } else {
    // Else it must be a MGATHER.
    auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
    Index = MGN->getIndex();
    Mask = MGN->getMask();
    PassThru = MGN->getPassThru();
    LoadExtType = MGN->getExtensionType();
  }
 
  MVT IndexVT = Index.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
         "Unexpected VTs!");
  assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
  // Targets have to explicitly opt-in for extending vector loads.
  assert(LoadExtType == ISD::NON_EXTLOAD &&
         "Unexpected extending MGATHER/VP_GATHER");
 
  // If the mask is known to be all ones, optimize to an unmasked intrinsic;
  // the selection of the masked intrinsics doesn't do this for us.
  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
                               ContainerVT.getVectorElementCount());
 
    Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
 
    if (!IsUnmasked) {
      MVT MaskVT = getMaskTypeFor(ContainerVT);
      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
      PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
    }
  }
 
  if (!VL)
    VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
 
  if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
    IndexVT = IndexVT.changeVectorElementType(XLenVT);
    Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
  }
 
  unsigned IntID =
      IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
  SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
  if (IsUnmasked)
    Ops.push_back(DAG.getUNDEF(ContainerVT));
  else
    Ops.push_back(PassThru);
  Ops.push_back(BasePtr);
  Ops.push_back(Index);
  if (!IsUnmasked)
    Ops.push_back(Mask);
  Ops.push_back(VL);
  if (!IsUnmasked)
    Ops.push_back(DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT));
 
  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
  SDValue Result =
      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
  Chain = Result.getValue(1);
 
  if (VT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
 
  return DAG.getMergeValues({Result, Chain}, DL);
}
 
// Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
// matched to a RVV indexed store. The RVV indexed store instructions only
// support the "unsigned unscaled" addressing mode; indices are implicitly
// zero-extended or truncated to XLEN and are treated as byte offsets. Any
// signed or scaled indexing is extended to the XLEN value type and scaled
// accordingly.
SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc DL(Op);
  const auto *MemSD = cast<MemSDNode>(Op.getNode());
  EVT MemVT = MemSD->getMemoryVT();
  MachineMemOperand *MMO = MemSD->getMemOperand();
  SDValue Chain = MemSD->getChain();
  SDValue BasePtr = MemSD->getBasePtr();
 
  [[maybe_unused]] bool IsTruncatingStore = false;
  SDValue Index, Mask, Val, VL;
 
  if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
    Index = VPSN->getIndex();
    Mask = VPSN->getMask();
    Val = VPSN->getValue();
    VL = VPSN->getVectorLength();
    // VP doesn't support truncating stores.
    IsTruncatingStore = false;
  } else {
    // Else it must be a MSCATTER.
    auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
    Index = MSN->getIndex();
    Mask = MSN->getMask();
    Val = MSN->getValue();
    IsTruncatingStore = MSN->isTruncatingStore();
  }
 
  MVT VT = Val.getSimpleValueType();
  MVT IndexVT = Index.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
         "Unexpected VTs!");
  assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
  // Targets have to explicitly opt-in for extending vector loads and
  // truncating vector stores.
  assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
 
  // If the mask is known to be all ones, optimize to an unmasked intrinsic;
  // the selection of the masked intrinsics doesn't do this for us.
  bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
                               ContainerVT.getVectorElementCount());
 
    Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
    Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
 
    if (!IsUnmasked) {
      MVT MaskVT = getMaskTypeFor(ContainerVT);
      Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
    }
  }
 
  if (!VL)
    VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
 
  if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
    IndexVT = IndexVT.changeVectorElementType(XLenVT);
    Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
  }
 
  unsigned IntID =
      IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
  SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
  Ops.push_back(Val);
  Ops.push_back(BasePtr);
  Ops.push_back(Index);
  if (!IsUnmasked)
    Ops.push_back(Mask);
  Ops.push_back(VL);
 
  return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
                                 DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
}
 
SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
                                               SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::frm, DL, XLenVT);
  SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
  SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
 
  // Encoding used for rounding mode in RISC-V differs from that used in
  // FLT_ROUNDS. To convert it the RISC-V rounding mode is used as an index in a
  // table, which consists of a sequence of 4-bit fields, each representing
  // corresponding FLT_ROUNDS mode.
  static const int Table =
      (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
      (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
      (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
      (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
      (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
 
  SDValue Shift =
      DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
  SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
                                DAG.getConstant(Table, DL, XLenVT), Shift);
  SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
                               DAG.getConstant(7, DL, XLenVT));
 
  return DAG.getMergeValues({Masked, Chain}, DL);
}
 
SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
                                               SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue RMValue = Op->getOperand(1);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::frm, DL, XLenVT);
 
  // Encoding used for rounding mode in RISC-V differs from that used in
  // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
  // a table, which consists of a sequence of 4-bit fields, each representing
  // corresponding RISC-V mode.
  static const unsigned Table =
      (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
      (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
      (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
      (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
      (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
 
  RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue);
 
  SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
                              DAG.getConstant(2, DL, XLenVT));
  SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
                                DAG.getConstant(Table, DL, XLenVT), Shift);
  RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
                        DAG.getConstant(0x7, DL, XLenVT));
  return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
                     RMValue);
}
 
SDValue RISCVTargetLowering::lowerGET_FPENV(SDValue Op,
                                            SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
  SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
  return DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
}
 
SDValue RISCVTargetLowering::lowerSET_FPENV(SDValue Op,
                                            SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue EnvValue = Op->getOperand(1);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
 
  EnvValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EnvValue);
  return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
                     EnvValue);
}
 
SDValue RISCVTargetLowering::lowerRESET_FPENV(SDValue Op,
                                              SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue EnvValue = DAG.getRegister(RISCV::X0, XLenVT);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
 
  return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
                     EnvValue);
}
 
const uint64_t ModeMask64 = ~RISCVExceptFlags::ALL;
const uint32_t ModeMask32 = ~RISCVExceptFlags::ALL;
 
SDValue RISCVTargetLowering::lowerGET_FPMODE(SDValue Op,
                                             SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
  SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
  SDValue Result = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
  Chain = Result.getValue(1);
  return DAG.getMergeValues({Result, Chain}, DL);
}
 
SDValue RISCVTargetLowering::lowerSET_FPMODE(SDValue Op,
                                             SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue EnvValue = Op->getOperand(1);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
  SDValue ModeMask = DAG.getConstant(ModeMaskValue, DL, XLenVT);
 
  EnvValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EnvValue);
  EnvValue = DAG.getNode(ISD::AND, DL, XLenVT, EnvValue, ModeMask);
  Chain = DAG.getNode(RISCVISD::CLEAR_CSR, DL, MVT::Other, Chain, SysRegNo,
                      ModeMask);
  return DAG.getNode(RISCVISD::SET_CSR, DL, MVT::Other, Chain, SysRegNo,
                     EnvValue);
}
 
SDValue RISCVTargetLowering::lowerRESET_FPMODE(SDValue Op,
                                               SelectionDAG &DAG) const {
  const MVT XLenVT = Subtarget.getXLenVT();
  const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
  SDValue ModeMask = DAG.getConstant(ModeMaskValue, DL, XLenVT);
 
  return DAG.getNode(RISCVISD::CLEAR_CSR, DL, MVT::Other, Chain, SysRegNo,
                     ModeMask);
}
 
SDValue RISCVTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
                                               SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
 
  bool isRISCV64 = Subtarget.is64Bit();
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
  int FI = MF.getFrameInfo().CreateFixedObject(isRISCV64 ? 8 : 4, 0, false);
  return DAG.getFrameIndex(FI, PtrVT);
}
 
// Returns the opcode of the target-specific SDNode that implements the 32-bit
// form of the given Opcode.
static unsigned getRISCVWOpcode(unsigned Opcode) {
  switch (Opcode) {
  default:
    llvm_unreachable("Unexpected opcode");
  case ISD::SHL:
    return RISCVISD::SLLW;
  case ISD::SRA:
    return RISCVISD::SRAW;
  case ISD::SRL:
    return RISCVISD::SRLW;
  case ISD::SDIV:
    return RISCVISD::DIVW;
  case ISD::UDIV:
    return RISCVISD::DIVUW;
  case ISD::UREM:
    return RISCVISD::REMUW;
  case ISD::ROTL:
    return RISCVISD::ROLW;
  case ISD::ROTR:
    return RISCVISD::RORW;
  }
}
 
// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
// node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
// otherwise be promoted to i64, making it difficult to select the
// SLLW/DIVUW/.../*W later one because the fact the operation was originally of
// type i8/i16/i32 is lost.
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
                                   unsigned ExtOpc = ISD::ANY_EXTEND) {
  SDLoc DL(N);
  unsigned WOpcode = getRISCVWOpcode(N->getOpcode());
  SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
  SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
  SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
  // ReplaceNodeResults requires we maintain the same type for the return value.
  return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
}
 
// Converts the given 32-bit operation to a i64 operation with signed extension
// semantic to reduce the signed extension instructions.
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
  SDLoc DL(N);
  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
  SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
  SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
                               DAG.getValueType(MVT::i32));
  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
}
 
// Zero-extend a 32-bit packed vector to the 64-bit packed type WideVT,
// clearing the upper lanes.
static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
                                          SDValue V, MVT WideVT) {
  SDValue Wide =
      DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, DAG.getBitcast(MVT::i32, V));
  return DAG.getBitcast(WideVT, Wide);
}
 
void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
                                             SmallVectorImpl<SDValue> &Results,
                                             SelectionDAG &DAG) const {
  SDLoc DL(N);
  switch (N->getOpcode()) {
  default:
    llvm_unreachable("Don't know how to custom type legalize this operation!");
  case ISD::STRICT_FP_TO_SINT:
  case ISD::STRICT_FP_TO_UINT:
  case ISD::FP_TO_SINT:
  case ISD::FP_TO_UINT: {
    bool IsStrict = N->isStrictFPOpcode();
    bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
                    N->getOpcode() == ISD::STRICT_FP_TO_SINT;
    SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
 
    // On RV32 only f16 is handled here: it is the only FP type whose finite
    // values always fit in an i32 after truncation towards zero, so convert in
    // i32 and extend instead of calling __fix(un)hfdi, which compiler-rt does
    // not provide. Out of range values and NaN are poison, so the fcvt clamping
    // is acceptable. Without Zfh/Zhinx the i32 conversion promotes f16 to f32.
    if (!Subtarget.is64Bit()) {
      assert(N->getValueType(0) == MVT::i64 &&
             "Unexpected custom legalisation");
      if (Op0.getValueType() != MVT::f16)
        return;
      SDValue Cvt;
      if (IsStrict) {
        Cvt = DAG.getNode(IsSigned ? ISD::STRICT_FP_TO_SINT
                                   : ISD::STRICT_FP_TO_UINT,
                          DL, {MVT::i32, MVT::Other}, {N->getOperand(0), Op0});
      } else {
        Cvt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL,
                          MVT::i32, Op0);
      }
      Results.push_back(DAG.getNode(
          IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL, MVT::i64, Cvt));
      if (IsStrict)
        Results.push_back(Cvt.getValue(1));
      return;
    }
 
    assert(N->getValueType(0) == MVT::i32 && "Unexpected custom legalisation");
    if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
        TargetLowering::TypeSoftenFloat) {
      if (!isTypeLegal(Op0.getValueType()))
        return;
      if (IsStrict) {
        SDValue Chain = N->getOperand(0);
        // In absence of Zfh, promote f16 to f32, then convert.
        if (Op0.getValueType() == MVT::f16 &&
            !Subtarget.hasStdExtZfhOrZhinx()) {
          Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                            {Chain, Op0});
          Chain = Op0.getValue(1);
        }
        unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
                                : RISCVISD::STRICT_FCVT_WU_RV64;
        SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
        SDValue Res = DAG.getNode(
            Opc, DL, VTs, Chain, Op0,
            DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
        Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
        Results.push_back(Res.getValue(1));
        return;
      }
      // For bf16, or f16 in absence of Zfh, promote [b]f16 to f32 and then
      // convert.
      if ((Op0.getValueType() == MVT::f16 &&
           !Subtarget.hasStdExtZfhOrZhinx()) ||
          Op0.getValueType() == MVT::bf16)
        Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
 
      unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
      SDValue Res =
          DAG.getNode(Opc, DL, MVT::i64, Op0,
                      DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    // If the FP type needs to be softened, emit a library call using the 'si'
    // version. If we left it to default legalization we'd end up with 'di'. If
    // the FP type doesn't need to be softened just let generic type
    // legalization promote the result type.
    RTLIB::Libcall LC;
    if (IsSigned)
      LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
    else
      LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
    MakeLibCallOptions CallOptions;
    EVT OpVT = Op0.getValueType();
    CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
    SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
    SDValue Result;
    std::tie(Result, Chain) =
        makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
    Results.push_back(Result);
    if (IsStrict)
      Results.push_back(Chain);
    break;
  }
  case ISD::LROUND: {
    SDValue Op0 = N->getOperand(0);
    EVT Op0VT = Op0.getValueType();
    if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
        TargetLowering::TypeSoftenFloat) {
      if (!isTypeLegal(Op0VT))
        return;
 
      // In absence of Zfh, promote f16 to f32, then convert.
      if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx())
        Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
 
      SDValue Res =
          DAG.getNode(RISCVISD::FCVT_W_RV64, DL, MVT::i64, Op0,
                      DAG.getTargetConstant(RISCVFPRndMode::RMM, DL, MVT::i64));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    // If the FP type needs to be softened, emit a library call to lround. We'll
    // need to truncate the result. We assume any value that doesn't fit in i32
    // is allowed to return an unspecified value.
    RTLIB::Libcall LC = RTLIB::getLROUND(Op0.getValueType());
    assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected FP type for LROUND!");
    MakeLibCallOptions CallOptions;
    EVT OpVT = Op0.getValueType();
    CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64);
    SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
    Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
    Results.push_back(Result);
    break;
  }
  case ISD::READCYCLECOUNTER:
  case ISD::READSTEADYCOUNTER: {
    assert(!Subtarget.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
                                   "has custom type legalization on riscv32");
 
    SDValue LoCounter, HiCounter;
    MVT XLenVT = Subtarget.getXLenVT();
    if (N->getOpcode() == ISD::READCYCLECOUNTER) {
      LoCounter = DAG.getTargetConstant(RISCVSysReg::cycle, DL, XLenVT);
      HiCounter = DAG.getTargetConstant(RISCVSysReg::cycleh, DL, XLenVT);
    } else {
      LoCounter = DAG.getTargetConstant(RISCVSysReg::time, DL, XLenVT);
      HiCounter = DAG.getTargetConstant(RISCVSysReg::timeh, DL, XLenVT);
    }
    SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
    SDValue RCW = DAG.getNode(RISCVISD::READ_COUNTER_WIDE, DL, VTs,
                              N->getOperand(0), LoCounter, HiCounter);
 
    Results.push_back(
        DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
    Results.push_back(RCW.getValue(2));
    break;
  }
  case ISD::LOAD: {
    if (!ISD::isNON_EXTLoad(N))
      return;
 
    // Use a SEXTLOAD instead of the default EXTLOAD. Similar to the
    // sext_inreg we emit for ADD/SUB/MUL/SLLI.
    LoadSDNode *Ld = cast<LoadSDNode>(N);
 
    if (N->getValueType(0) == MVT::i64) {
      assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
             "Unexpected custom legalisation");
 
      if (Ld->getAlign() < Subtarget.getZilsdAlign())
        return;
 
      SDLoc DL(N);
      SDValue Result = DAG.getMemIntrinsicNode(
          RISCVISD::LD_RV32, DL,
          DAG.getVTList({MVT::i32, MVT::i32, MVT::Other}),
          {Ld->getChain(), Ld->getBasePtr()}, MVT::i64, Ld->getMemOperand());
      SDValue Lo = Result.getValue(0);
      SDValue Hi = Result.getValue(1);
      SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
      Results.append({Pair, Result.getValue(2)});
      return;
    }
 
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
 
    SDLoc dl(N);
    SDValue Res = DAG.getExtLoad(ISD::SEXTLOAD, dl, MVT::i64, Ld->getChain(),
                                 Ld->getBasePtr(), Ld->getMemoryVT(),
                                 Ld->getMemOperand());
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Res));
    Results.push_back(Res.getValue(1));
    return;
  }
  case ISD::MUL: {
    unsigned Size = N->getSimpleValueType(0).getSizeInBits();
    unsigned XLen = Subtarget.getXLen();
    if (Size > XLen) {
      // This multiply needs to be expanded, try to use MULH+MUL or WMUL if
      // possible. We duplicate the default legalization to
      // MULHU/MULHS/UMUL_LOHI/SMUL_LOHI to minimize the number of calls to
      // MaskedValueIsZero and ComputeNumSignBits
      // FIXME: Should we have a target independent MULHSU/WMULSU node? Are
      // there are other targets that could use it?
      assert(Size == (XLen * 2) && "Unexpected custom legalisation");
 
      auto MakeMULPair = [&](SDValue L, SDValue R, unsigned HighOpc,
                             unsigned LoHiOpc) {
        MVT XLenVT = Subtarget.getXLenVT();
        L = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, L);
        R = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, R);
        SDValue Lo, Hi;
        if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
          SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
          Lo = DAG.getNode(LoHiOpc, DL, VTs, L, R);
          Hi = Lo.getValue(1);
        } else {
          Lo = DAG.getNode(ISD::MUL, DL, XLenVT, L, R);
          Hi = DAG.getNode(HighOpc, DL, XLenVT, L, R);
        }
        return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
      };
 
      SDValue LHS = N->getOperand(0);
      SDValue RHS = N->getOperand(1);
 
      APInt HighMask = APInt::getHighBitsSet(Size, XLen);
      bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
      bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
      if (LHSIsU && RHSIsU) {
        Results.push_back(MakeMULPair(LHS, RHS, ISD::MULHU, ISD::UMUL_LOHI));
        return;
      }
 
      bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
      bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
      if (LHSIsS && RHSIsS)
        Results.push_back(MakeMULPair(LHS, RHS, ISD::MULHS, ISD::SMUL_LOHI));
      else if (RHSIsU && LHSIsS)
        Results.push_back(
            MakeMULPair(LHS, RHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
      else if (LHSIsU && RHSIsS)
        Results.push_back(
            MakeMULPair(RHS, LHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
 
      return;
    }
    [[fallthrough]];
  }
  case ISD::ADD:
  case ISD::SUB:
    if (N->getValueType(0) == MVT::i64) {
      assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
             "Unexpected custom legalisation");
 
      // Expand to ADDD/SUBD.
      auto [LHSLo, LHSHi] =
          DAG.SplitScalar(N->getOperand(0), DL, MVT::i32, MVT::i32);
      auto [RHSLo, RHSHi] =
          DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32);
      unsigned Opc =
          N->getOpcode() == ISD::ADD ? RISCVISD::ADDD : RISCVISD::SUBD;
      SDValue Res = DAG.getNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::i32),
                                LHSLo, LHSHi, RHSLo, RHSHi);
      Res = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1));
      Results.push_back(Res);
      return;
    }
 
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
    Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
    break;
  case ISD::TRUNCATE: {
    MVT VT = N->getSimpleValueType(0);
    assert(VT.isFixedLengthVector() && Subtarget.hasStdExtP() &&
           Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
           "Unexpected custom legalisation");
 
    // v4i16->v4i8 and v2i32->v2i16 truncates aren't legal on their own, but
    // the widened result type is. Bitcast the operand to the widened result
    // type and use a shuffle to select the low half of each element, which
    // gets matched to a P-extension packed narrowing convert
    // (unzip8p/unzip16p).
 
    SDValue Op0 = N->getOperand(0);
 
    // Input should be a 64-bit vector.
    if (!Op0.getValueType().is64BitVector())
      break;
 
    MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
 
    unsigned NumElts = VT.getVectorNumElements();
    SmallVector<int, 8> ShuffleMask(WidenVT.getVectorNumElements(), -1);
    for (unsigned i = 0; i != NumElts; ++i)
      ShuffleMask[i] = i * 2;
 
    SDValue Src = DAG.getBitcast(WidenVT, Op0);
    Results.push_back(DAG.getVectorShuffle(WidenVT, DL, Src,
                                           DAG.getUNDEF(WidenVT), ShuffleMask));
    return;
  }
  case ISD::SHL:
  case ISD::SRA:
  case ISD::SRL: {
    EVT VT = N->getValueType(0);
    if (VT.isFixedLengthVector() && Subtarget.hasStdExtP()) {
      assert(Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
             "Unexpected vector type for P-extension shift");
 
      // If shift amount is a splat, don't scalarize - let normal widening
      // and SIMD patterns handle it (pslli.h, psrli.h, etc.)
      SDValue ShiftAmt = N->getOperand(1);
      if (DAG.isSplatValue(ShiftAmt, /*AllowUndefs=*/true))
        break;
 
      EVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT);
      unsigned WidenNumElts = WidenVT.getVectorNumElements();
      // Unroll with OrigNumElts operations, padding result to WidenNumElts
      SDValue Res = DAG.UnrollVectorOp(N, WidenNumElts);
      Results.push_back(Res);
      break;
    }
 
    if (VT == MVT::i64) {
      assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
             "Unexpected custom legalisation");
 
      SDValue LHS = N->getOperand(0);
      SDValue ShAmt = N->getOperand(1);
 
      unsigned WideOpc = 0;
      APInt HighMask = APInt::getHighBitsSet(64, 32);
      if (DAG.MaskedValueIsZero(LHS, HighMask))
        WideOpc = RISCVISD::WSLL;
      else if (DAG.ComputeMaxSignificantBits(LHS) <= 32)
        WideOpc = RISCVISD::WSLA;
 
      if (WideOpc) {
        SDValue Res =
            DAG.getNode(WideOpc, DL, DAG.getVTList(MVT::i32, MVT::i32),
                        DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LHS),
                        DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ShAmt));
        Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0),
                                      Res, Res.getValue(1)));
        return;
      }
 
      // Only handle constant shifts < 32. Non-constant shifts are handled by
      // lowerShiftLeftParts/lowerShiftRightParts, and shifts >= 32 use default
      // legalization.
      auto *ShAmtC = dyn_cast<ConstantSDNode>(N->getOperand(1));
      if (!ShAmtC || ShAmtC->getZExtValue() >= 32)
        break;
 
      auto [Lo, Hi] = DAG.SplitScalar(LHS, DL, MVT::i32, MVT::i32);
 
      // If the shift amount operand is coming from a vector legalization it may
      // have an illegal type.
      if (ShAmt.getValueType() != MVT::i32)
        ShAmt = DAG.getZExtOrTrunc(ShAmt, DL, MVT::i32);
 
      SDValue LoRes, HiRes;
      if (N->getOpcode() == ISD::SHL) {
        // Lo = slli Lo, shamt
        // Hi = nsrli {Hi, Lo}, (32 - shamt)
        uint64_t ShAmtVal = ShAmtC->getZExtValue();
        LoRes = DAG.getNode(ISD::SHL, DL, MVT::i32, Lo, ShAmt);
        HiRes = DAG.getNode(RISCVISD::NSRL, DL, MVT::i32, Lo, Hi,
                            DAG.getConstant(32 - ShAmtVal, DL, MVT::i32));
      } else {
        bool IsSRA = N->getOpcode() == ISD::SRA;
        LoRes = DAG.getNode(IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL,
                            MVT::i32, Lo, Hi, ShAmt);
        HiRes =
            DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, DL, MVT::i32, Hi, ShAmt);
      }
      SDValue Res = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, LoRes, HiRes);
      Results.push_back(Res);
      return;
    }
 
    assert(VT == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
    if (N->getOperand(1).getOpcode() != ISD::Constant) {
      // If we can use a BSET instruction, allow default promotion to apply.
      if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
          isOneConstant(N->getOperand(0)))
        break;
      Results.push_back(customLegalizeToWOp(N, DAG));
      break;
    }
 
    // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
    // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
    // shift amount.
    if (N->getOpcode() == ISD::SHL) {
      SDLoc DL(N);
      SDValue NewOp0 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
      SDValue NewOp1 =
          DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
      SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
      SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
                                   DAG.getValueType(MVT::i32));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
    }
 
    break;
  }
  case ISD::ROTL:
  case ISD::ROTR:
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
    assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
            Subtarget.hasVendorXTHeadBb()) &&
           "Unexpected custom legalization");
    if (!isa<ConstantSDNode>(N->getOperand(1)) &&
        !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
      return;
    Results.push_back(customLegalizeToWOp(N, DAG));
    break;
  case ISD::CTTZ:
  case ISD::CTTZ_ZERO_POISON:
  case ISD::CTLZ:
  case ISD::CTLZ_ZERO_POISON:
  case ISD::CTLS: {
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
 
    SDValue NewOp0 =
        DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
    unsigned Opc;
    switch (N->getOpcode()) {
    default: llvm_unreachable("Unexpected opcode");
    case ISD::CTTZ:
    case ISD::CTTZ_ZERO_POISON:
      Opc = RISCVISD::CTZW;
      break;
    case ISD::CTLZ:
    case ISD::CTLZ_ZERO_POISON:
      Opc = RISCVISD::CLZW;
      break;
    case ISD::CTLS:
      Opc = RISCVISD::CLSW;
      break;
    }
 
    SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
    return;
  }
  case ISD::SDIV:
  case ISD::UDIV:
  case ISD::UREM: {
    MVT VT = N->getSimpleValueType(0);
    assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
           Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
           "Unexpected custom legalisation");
    // Don't promote division/remainder by constant since we should expand those
    // to multiply by magic constant.
    AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
    if (N->getOperand(1).getOpcode() == ISD::Constant &&
        !isIntDivCheap(N->getValueType(0), Attr))
      return;
 
    // If the input is i32, use ANY_EXTEND since the W instructions don't read
    // the upper 32 bits. For other types we need to sign or zero extend
    // based on the opcode.
    unsigned ExtOpc = ISD::ANY_EXTEND;
    if (VT != MVT::i32)
      ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
                                           : ISD::ZERO_EXTEND;
 
    Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
    break;
  }
  case ISD::SADDO:
  case ISD::SSUBO: {
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
 
    // This is similar to the default legalization, but we return the
    // sext_inreg instead of the add/sub.
    bool IsAdd = N->getOpcode() == ISD::SADDO;
    SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
    SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
    SDValue Op =
        DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
    SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Op,
                              DAG.getValueType(MVT::i32));
 
    SDValue Overflow;
 
    // If the RHS is a constant, we can simplify ConditionRHS below. Otherwise
    // use the default legalization.
    if (IsAdd && isa<ConstantSDNode>(N->getOperand(1))) {
      SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
 
      // For an addition, the result should be less than one of the operands
      // (LHS) if and only if the other operand (RHS) is negative, otherwise
      // there will be overflow.
      EVT OType = N->getValueType(1);
      SDValue ResultLowerThanLHS =
          DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT);
      SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT);
 
      Overflow =
          DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS);
    } else {
      Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, Op, ISD::SETNE);
    }
 
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
    Results.push_back(Overflow);
    return;
  }
  case ISD::UADDO:
  case ISD::USUBO: {
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
    bool IsAdd = N->getOpcode() == ISD::UADDO;
    // Create an ADDW or SUBW.
    SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
    SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
    SDValue Res =
        DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
    Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
                      DAG.getValueType(MVT::i32));
 
    SDValue Overflow;
    if (IsAdd && isOneConstant(RHS)) {
      // Special case uaddo X, 1 overflowed if the addition result is 0.
      // The general case (X + C) < C is not necessarily beneficial. Although we
      // reduce the live range of X, we may introduce the materialization of
      // constant C, especially when the setcc result is used by branch. We have
      // no compare with constant and branch instructions.
      Overflow = DAG.getSetCC(DL, N->getValueType(1), Res,
                              DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ);
    } else if (IsAdd && isAllOnesConstant(RHS)) {
      // Special case uaddo X, -1 overflowed if X != 0.
      Overflow = DAG.getSetCC(DL, N->getValueType(1), N->getOperand(0),
                              DAG.getConstant(0, DL, MVT::i32), ISD::SETNE);
    } else {
      // Sign extend the LHS and perform an unsigned compare with the ADDW
      // result. Since the inputs are sign extended from i32, this is equivalent
      // to comparing the lower 32 bits.
      LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
      Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
                              IsAdd ? ISD::SETULT : ISD::SETUGT);
    }
 
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
    Results.push_back(Overflow);
    return;
  }
  case ISD::UADDSAT:
  case ISD::USUBSAT:
  case ISD::SADDSAT:
  case ISD::SSUBSAT: {
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
 
    if (Subtarget.hasStdExtP()) {
      // On RV64, map scalar i32 saturating add/sub through lane 0 of a packed
      // v2i32 operation so we can select ps*.w instructions.
      SDValue LHS = DAG.getNode(
          ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)));
      SDValue RHS = DAG.getNode(
          ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)));
      SDValue VecRes = DAG.getNode(N->getOpcode(), DL, MVT::v2i32, LHS, RHS);
      SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
      Results.push_back(
          DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, VecRes, Zero));
      return;
    }
 
    assert(!Subtarget.hasStdExtZbb() && "Unexpected custom legalisation");
    Results.push_back(expandAddSubSat(N, DAG));
    return;
  }
  case ISD::ABS:
  case ISD::ABS_MIN_POISON: {
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
           "Unexpected custom legalisation");
 
    if (Subtarget.hasStdExtP()) {
      SDValue Src =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
      SDValue Abs = DAG.getNode(RISCVISD::ABSW, DL, MVT::i64, Src);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
      return;
    }
 
    if (Subtarget.hasStdExtZbb()) {
      // Emit a special node that will be expanded to NEGW+MAX at isel.
      // This allows us to remember that the result is sign extended. Expanding
      // to NEGW+MAX here requires a Freeze which breaks ComputeNumSignBits.
      SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64,
                                N->getOperand(0));
      SDValue Abs = DAG.getNode(RISCVISD::NEGW_MAX, DL, MVT::i64, Src);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
      return;
    }
 
    // Expand abs to Y = (sraiw X, 31); subw(xor(X, Y), Y)
    SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
 
    // Freeze the source so we can increase it's use count.
    Src = DAG.getFreeze(Src);
 
    // Copy sign bit to all bits using the sraiw pattern.
    SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src,
                                   DAG.getValueType(MVT::i32));
    SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill,
                           DAG.getConstant(31, DL, MVT::i64));
 
    SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill);
    NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill);
 
    // NOTE: The result is only required to be anyextended, but sext is
    // consistent with type legalization of sub.
    NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes,
                         DAG.getValueType(MVT::i32));
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
    return;
  }
  case ISD::BITCAST: {
    EVT VT = N->getValueType(0);
    assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
    SDValue Op0 = N->getOperand(0);
    EVT Op0VT = Op0.getValueType();
    MVT XLenVT = Subtarget.getXLenVT();
    if (VT == MVT::i16 &&
        ((Op0VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
         (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
      SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
    } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
               Subtarget.hasStdExtFOrZfinx()) {
      SDValue FPConv =
          DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
    } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() &&
               Subtarget.hasStdExtDOrZdinx()) {
      SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL,
                                   DAG.getVTList(MVT::i32, MVT::i32), Op0);
      SDValue Lo = NewReg.getValue(0);
      SDValue Hi = NewReg.getValue(1);
      // For big-endian, swap the order when building the i64 pair.
      if (!Subtarget.isLittleEndian())
        std::swap(Lo, Hi);
      SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
      Results.push_back(RetReg);
    } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
               isTypeLegal(Op0VT)) {
      // Custom-legalize bitcasts from fixed-length vector types to illegal
      // scalar types in order to improve codegen. Bitcast the vector to a
      // one-element vector type whose element type is the same as the result
      // type, and extract the first element.
      EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
      if (isTypeLegal(BVT)) {
        SDValue BVec = DAG.getBitcast(BVT, Op0);
        Results.push_back(DAG.getExtractVectorElt(DL, VT, BVec, 0));
      }
    }
    break;
  }
  case ISD::BITREVERSE: {
    assert(N->getValueType(0) == MVT::i8 && Subtarget.hasStdExtZbkb() &&
           "Unexpected custom legalisation");
    MVT XLenVT = Subtarget.getXLenVT();
    SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
    SDValue NewRes = DAG.getNode(RISCVISD::BREV8, DL, XLenVT, NewOp);
    // ReplaceNodeResults requires we maintain the same type for the return
    // value.
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, NewRes));
    break;
  }
  case RISCVISD::BREV8:
  case RISCVISD::ORC_B: {
    MVT VT = N->getSimpleValueType(0);
    MVT XLenVT = Subtarget.getXLenVT();
    assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
           "Unexpected custom legalisation");
    assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) ||
            (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) &&
           "Unexpected extension");
    SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
    SDValue NewRes = DAG.getNode(N->getOpcode(), DL, XLenVT, NewOp);
    // ReplaceNodeResults requires we maintain the same type for the return
    // value.
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes));
    break;
  }
  case RISCVISD::ASUB:
  case RISCVISD::ASUBU:
  case RISCVISD::MULHSU:
  case RISCVISD::MULHR:
  case RISCVISD::MULHRU:
  case RISCVISD::MULHRSU: {
    MVT VT = N->getSimpleValueType(0);
    SDValue Op0 = N->getOperand(0);
    SDValue Op1 = N->getOperand(1);
    unsigned Opcode = N->getOpcode();
    // PMULH* variants don't support i8
    [[maybe_unused]] bool IsMulH =
        Opcode == RISCVISD::MULHSU || Opcode == RISCVISD::MULHR ||
        Opcode == RISCVISD::MULHRU || Opcode == RISCVISD::MULHRSU;
    assert(VT == MVT::v2i16 || (!IsMulH && VT == MVT::v4i8));
    MVT NewVT = MVT::v4i16;
    if (VT == MVT::v4i8)
      NewVT = MVT::v8i8;
    SDValue Undef = DAG.getUNDEF(VT);
    Op0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, {Op0, Undef});
    Op1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, {Op1, Undef});
    Results.push_back(DAG.getNode(Opcode, DL, NewVT, {Op0, Op1}));
    return;
  }
  case ISD::EXTRACT_VECTOR_ELT: {
    // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
    // type is illegal (currently only vXi64 RV32).
    // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
    // transferred to the destination register. We issue two of these from the
    // upper- and lower- halves of the SEW-bit vector element, slid down to the
    // first element.
    SDValue Vec = N->getOperand(0);
    SDValue Idx = N->getOperand(1);
 
    // The vector type hasn't been legalized yet so we can't issue target
    // specific nodes if it needs legalization.
    // FIXME: We would manually legalize if it's important.
    if (!isTypeLegal(Vec.getValueType()))
      return;
 
    MVT VecVT = Vec.getSimpleValueType();
 
    assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
           VecVT.getVectorElementType() == MVT::i64 &&
           "Unexpected EXTRACT_VECTOR_ELT legalization");
 
    // If this is a fixed vector, we need to convert it to a scalable vector.
    MVT ContainerVT = VecVT;
    if (VecVT.isFixedLengthVector()) {
      ContainerVT = getContainerForFixedLengthVector(VecVT);
      Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
    }
 
    MVT XLenVT = Subtarget.getXLenVT();
 
    // Use a VL of 1 to avoid processing more elements than we need.
    auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
 
    // Unless the index is known to be 0, we must slide the vector down to get
    // the desired element into index 0.
    if (!isNullConstant(Idx)) {
      Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
                          DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
    }
 
    // Extract the lower XLEN bits of the correct vector element.
    SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
 
    // To extract the upper XLEN bits of the vector element, shift the first
    // element right by 32 bits and re-extract the lower XLEN bits.
    SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                     DAG.getUNDEF(ContainerVT),
                                     DAG.getConstant(32, DL, XLenVT), VL);
    SDValue LShr32 =
        DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
                    DAG.getUNDEF(ContainerVT), Mask, VL);
 
    SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
 
    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
    break;
  }
  case ISD::INTRINSIC_WO_CHAIN: {
    unsigned IntNo = N->getConstantOperandVal(0);
    switch (IntNo) {
    default:
      llvm_unreachable(
          "Don't know how to custom type legalize this intrinsic!");
    case Intrinsic::experimental_get_vector_length: {
      SDValue Res = lowerGetVectorLength(N, DAG, Subtarget);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_psext_h:
      reportFatalUsageError("unsupported llvm.riscv.psext intrinsic");
    case Intrinsic::riscv_pzext_h:
      reportFatalUsageError("unsupported llvm.riscv.pzext intrinsic");
    case Intrinsic::riscv_psext_b:
    case Intrinsic::riscv_pzext_b: {
      bool IsSExt = IntNo == Intrinsic::riscv_psext_b;
      const char *UnsupportedMsg =
          IsSExt ? "unsupported llvm.riscv.psext intrinsic"
                 : "unsupported llvm.riscv.pzext intrinsic";
      EVT VT = N->getValueType(0);
      if (!Subtarget.is64Bit() || VT != MVT::v2i16)
        reportFatalUsageError(UnsupportedMsg);
 
      SDValue Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                N->getOperand(1), DAG.getUNDEF(VT));
      SDValue Res;
      if (IsSExt)
        Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::v4i16, Src,
                          DAG.getValueType(MVT::v4i8));
      else
        Res = lowerPZExt(Src, DL, DAG, Subtarget);
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_pmhacc:
    case Intrinsic::riscv_pmhracc:
    case Intrinsic::riscv_pmhaccu:
    case Intrinsic::riscv_pmhraccu:
    case Intrinsic::riscv_pmhaccsu:
    case Intrinsic::riscv_pmhraccsu: {
      EVT VT = N->getValueType(0);
      if (!Subtarget.is64Bit() || VT != MVT::v2i16)
        return;
 
      EVT WideVT = MVT::v4i16;
      SDValue Undef = DAG.getUNDEF(VT);
      SDValue Rd =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, N->getOperand(1), Undef);
      SDValue Rs1 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, N->getOperand(2), Undef);
      SDValue Rs2 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, N->getOperand(3), Undef);
      SDValue Res = DAG.getNode(getRVPMulHighAccumulateOpcode(IntNo), DL,
                                WideVT, Rd, Rs1, Rs2);
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_pmhacc_b0:
    case Intrinsic::riscv_pmhacc_b1:
    case Intrinsic::riscv_pmhaccsu_b0:
    case Intrinsic::riscv_pmhaccsu_b1: {
      // pmhacc.h.bXX operates on the whole register: v2i16 on RV32 and
      // v4i16 on RV64. On RV64 a v2i16 result widens to the packed v4i16
      // form and extracts the low half.
      EVT VT = N->getValueType(0);
      if (!Subtarget.is64Bit() || VT != MVT::v2i16)
        return;
 
      EVT WideVT = MVT::v4i16;
      SDValue Undef = DAG.getUNDEF(VT);
      SDValue Rd =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, N->getOperand(1), Undef);
      SDValue Rs1 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, N->getOperand(2), Undef);
      // Third operand is v4i8 - expand to v8i8
      SDValue Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8,
                                N->getOperand(3), DAG.getUNDEF(MVT::v4i8));
      SDValue Res = DAG.getNode(getRVPMulHighAccumulateByHalvesOpcode(IntNo),
                                DL, WideVT, Rd, Rs1, Rs2);
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_pm4add:
    case Intrinsic::riscv_pm2add:
    case Intrinsic::riscv_pm2add_x:
    case Intrinsic::riscv_pm4addu:
    case Intrinsic::riscv_pm2addu:
    case Intrinsic::riscv_pmq2add:
    case Intrinsic::riscv_pmqr2add:
    case Intrinsic::riscv_pm2sadd:
    case Intrinsic::riscv_pm2sadd_x:
    case Intrinsic::riscv_pm2sub:
    case Intrinsic::riscv_pm2sub_x:
    case Intrinsic::riscv_pm4addsu:
    case Intrinsic::riscv_pm2addsu: {
      MVT VT = N->getSimpleValueType(0);
      unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
      SDValue Rs1 = N->getOperand(1);
      SDValue Rs2 = N->getOperand(2);
 
      if (!Subtarget.is64Bit() && VT == MVT::i64) {
        SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG);
        Results.push_back(Res);
        return;
      }
 
      assert(Subtarget.is64Bit() && VT == MVT::i32 &&
             "Unexpected horizontal multiply legalization");
      MVT SrcVT = Rs1.getSimpleValueType();
      MVT WideSrcVT = SrcVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
      Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs1,
                        DAG.getUNDEF(SrcVT));
      Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs2,
                        DAG.getUNDEF(SrcVT));
      SDValue Wide = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2);
      Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Wide, 0));
      return;
    }
 
    case Intrinsic::riscv_pmul_00:
    case Intrinsic::riscv_pmul_01:
    case Intrinsic::riscv_pmul_11:
    case Intrinsic::riscv_pmulu_00:
    case Intrinsic::riscv_pmulu_01:
    case Intrinsic::riscv_pmulu_11:
    case Intrinsic::riscv_pmulsu_00:
    case Intrinsic::riscv_pmulsu_11: {
      MVT VT = N->getSimpleValueType(0);
      if (!Subtarget.is64Bit() || VT != MVT::v2i16)
        return;
 
      SDValue Undef = DAG.getUNDEF(MVT::v4i8);
      SDValue Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8,
                                N->getOperand(1), Undef);
      SDValue Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8,
                                N->getOperand(2), Undef);
      SDValue Res =
          DAG.getNode(getRVPMulHalvesOpcode(IntNo), DL, MVT::v4i16, Rs1, Rs2);
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_mul_00:
    case Intrinsic::riscv_mul_01:
    case Intrinsic::riscv_mul_11:
    case Intrinsic::riscv_mulu_00:
    case Intrinsic::riscv_mulu_01:
    case Intrinsic::riscv_mulu_11:
    case Intrinsic::riscv_mulsu_00:
    case Intrinsic::riscv_mulsu_11: {
      // mul.hXX exists only on RV32 and mul.wXX only on RV64; the other XLEN
      // has to build the product here.
      MVT VT = N->getSimpleValueType(0);
      MVT SrcVT = N->getOperand(1).getSimpleValueType();
      if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::i32 &&
          SrcVT == MVT::v2i16) {
        // The halfword product is the first element of the packed one.
        SDValue Undef = DAG.getUNDEF(SrcVT);
        SDValue Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                  N->getOperand(1), Undef);
        SDValue Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                  N->getOperand(2), Undef);
        SDValue Res =
            DAG.getNode(getRVPMulHalvesOpcode(IntNo), DL, MVT::v2i32, Rs1, Rs2);
        Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
        return;
      }
      if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::i64 &&
          SrcVT == MVT::v2i32) {
        auto [Opc, Rs1Lane, Rs2Lane] = getRVPWordMulPartsOpcodeAndLanes(IntNo);
        SDValue Rs1 =
            DAG.getExtractVectorElt(DL, MVT::i32, N->getOperand(1), Rs1Lane);
        SDValue Rs2 =
            DAG.getExtractVectorElt(DL, MVT::i32, N->getOperand(2), Rs2Lane);
        SDValue Res =
            DAG.getNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::i32), Rs1, Rs2);
        Results.push_back(
            DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1)));
        return;
      }
      reportFatalUsageError("unsupported llvm.riscv multiply-parts intrinsic");
    }
    case Intrinsic::riscv_pmulh_b0:
    case Intrinsic::riscv_pmulh_b1:
    case Intrinsic::riscv_pmulhsu_b0:
    case Intrinsic::riscv_pmulhsu_b1: {
      // A 32-bit result has no legal container on RV64; widen the result and
      // both sources to the 64-bit packed form.
      EVT VT = N->getValueType(0);
      if (!Subtarget.is64Bit() || VT != MVT::v2i16)
        return;
 
      SDValue Undef = DAG.getUNDEF(VT);
      SDValue Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                N->getOperand(1), Undef);
      SDValue Src2 = N->getOperand(2);
      SDValue Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Src2,
                                DAG.getUNDEF(Src2.getValueType()));
      SDValue Res = DAG.getNode(getRVPMulHighPartsOpcode(IntNo), DL, MVT::v4i16,
                                Rs1, Rs2);
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_mulh_h0:
    case Intrinsic::riscv_mulh_h1:
    case Intrinsic::riscv_mulhsu_h0:
    case Intrinsic::riscv_mulhsu_h1: {
      // mulh.h0/h1 exist only on RV32 and pmulh.w.h0/h1 only on RV64; the
      // other XLEN has to build the product here.
      MVT VT = N->getSimpleValueType(0);
      MVT SrcVT = N->getOperand(2).getSimpleValueType();
      if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::i32 &&
          SrcVT == MVT::v2i16) {
        // The halfword product is the first element of the packed one.
        SDValue Rd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
                                 N->getOperand(1));
        SDValue Undef = DAG.getUNDEF(SrcVT);
        SDValue Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                  N->getOperand(2), Undef);
        SDValue Res = DAG.getNode(getRVPMulHighPartsWOpcode(IntNo), DL,
                                  MVT::v2i32, Rd, Rs1);
        Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
        return;
      }
      reportFatalUsageError(
          "unsupported llvm.riscv multiply-high-parts intrinsic");
    }
    case Intrinsic::riscv_macc_00:
    case Intrinsic::riscv_macc_01:
    case Intrinsic::riscv_macc_11:
    case Intrinsic::riscv_maccu_00:
    case Intrinsic::riscv_maccu_01:
    case Intrinsic::riscv_maccu_11:
    case Intrinsic::riscv_maccsu_00:
    case Intrinsic::riscv_maccsu_11: {
      // macc.hXX exists only on RV32 and macc.wXX only on RV64; the other XLEN
      // has to build the product here.
      MVT VT = N->getSimpleValueType(0);
      MVT SrcVT = N->getOperand(2).getSimpleValueType();
      if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::i32 &&
          SrcVT == MVT::v2i16) {
        // Accumulate into the first element of the packed product.
        SDValue Undef = DAG.getUNDEF(SrcVT);
        SDValue Rd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
                                 N->getOperand(1));
        SDValue Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                  N->getOperand(2), Undef);
        SDValue Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                                  N->getOperand(3), Undef);
        SDValue Res = DAG.getNode(getRVPMulAccHalvesOpcode(IntNo), DL,
                                  MVT::v2i32, Rd, Rs1, Rs2);
        Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
        return;
      }
      if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::i64 &&
          SrcVT == MVT::v2i32) {
        auto [Opc, Rs1Lane, Rs2Lane] =
            getRVPWordMulPartsAccOpcodeAndLanes(IntNo);
        auto [RdLo, RdHi] =
            DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32);
        SDValue Rs1 =
            DAG.getExtractVectorElt(DL, MVT::i32, N->getOperand(2), Rs1Lane);
        SDValue Rs2 =
            DAG.getExtractVectorElt(DL, MVT::i32, N->getOperand(3), Rs2Lane);
        SDValue Res = DAG.getNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::i32),
                                  RdLo, RdHi, Rs1, Rs2);
        Results.push_back(
            DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1)));
        return;
      }
      reportFatalUsageError("unsupported llvm.riscv multiply-parts intrinsic");
    }
    case Intrinsic::riscv_paadd:
    case Intrinsic::riscv_paaddu:
    case Intrinsic::riscv_pasub:
    case Intrinsic::riscv_pasubu:
    case Intrinsic::riscv_pabd:
    case Intrinsic::riscv_pabdu:
    case Intrinsic::riscv_pas:
    case Intrinsic::riscv_psa:
    case Intrinsic::riscv_psas:
    case Intrinsic::riscv_pssa:
    case Intrinsic::riscv_paas:
    case Intrinsic::riscv_pasa:
    case Intrinsic::riscv_pmerge:
    case Intrinsic::riscv_pmulq:
    case Intrinsic::riscv_pmulqr:
    case Intrinsic::riscv_pmulhr:
    case Intrinsic::riscv_pmulhru:
    case Intrinsic::riscv_pmulhsu:
    case Intrinsic::riscv_pmulhrsu:
    case Intrinsic::riscv_psabs:
    case Intrinsic::riscv_psati:
    case Intrinsic::riscv_pusati: {
      EVT VT = N->getValueType(0);
      if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
        return;
 
      unsigned Opc;
      switch (IntNo) {
      case Intrinsic::riscv_paadd:
        Opc = ISD::AVGFLOORS;
        break;
      case Intrinsic::riscv_paaddu:
        Opc = ISD::AVGFLOORU;
        break;
      case Intrinsic::riscv_pasub:
        Opc = RISCVISD::ASUB;
        break;
      case Intrinsic::riscv_pasubu:
        Opc = RISCVISD::ASUBU;
        break;
      case Intrinsic::riscv_pabd:
        Opc = ISD::ABDS;
        break;
      case Intrinsic::riscv_pabdu:
        Opc = ISD::ABDU;
        break;
      case Intrinsic::riscv_psabs:
        Opc = RISCVISD::PSABS;
        break;
      case Intrinsic::riscv_pmulq:
        Opc = RISCVISD::MULQ;
        break;
      case Intrinsic::riscv_pmulqr:
        Opc = RISCVISD::MULQR;
        break;
      case Intrinsic::riscv_pmulhr:
      case Intrinsic::riscv_pmulhru:
      case Intrinsic::riscv_pmulhsu:
      case Intrinsic::riscv_pmulhrsu:
        Opc = getRVPMulHighOpcode(IntNo);
        break;
      default:
        // pas/psa/psas/pssa/paas/pasa, pmerge and psati/pusati: re-emit at the
        // widened type rather than lowering to a generic node.
        Opc = ISD::INTRINSIC_WO_CHAIN;
        break;
      }
 
      EVT WideVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
      SDValue Undef = DAG.getUNDEF(VT);
      SmallVector<SDValue, 4> Ops(N->ops());
      for (SDValue &Op : Ops) {
        if (Op.getValueType() == VT)
          Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, Op, Undef);
      }
      SDValue Res;
      if (Opc == ISD::INTRINSIC_WO_CHAIN)
        Res = DAG.getNode(Opc, DL, WideVT, Ops);
      else
        Res = DAG.getNode(Opc, DL, WideVT, ArrayRef(Ops).slice(1));
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_mulhr_i32:
    case Intrinsic::riscv_mulhru_u32:
    case Intrinsic::riscv_mulhsu_i32:
    case Intrinsic::riscv_mulhrsu_i32: {
      // RV64 has no scalar mulh instructions; reuse the packed pmulh.w
      // family on the low words, whose element 0 is the scalar product.
      MVT VT = N->getSimpleValueType(0);
      if (!Subtarget.is64Bit() || VT != MVT::i32)
        return;
      unsigned Opc = getRVScalarMulHighOpcode(IntNo);
      SDValue Rs1 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, N->getOperand(1));
      SDValue Rs2 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, N->getOperand(2));
      SDValue Res = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2);
      Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
      return;
    }
    case Intrinsic::riscv_pnclipp:
    case Intrinsic::riscv_pnclipup: {
      bool IsSigned = IntNo == Intrinsic::riscv_pnclipp;
      EVT VT = N->getValueType(0);
      if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
        return;
      unsigned Opc = IsSigned ? RISCVISD::PNCLIPP : RISCVISD::PNCLIPUP;
      SDValue Src1 = N->getOperand(1);
      SDValue Src2 = N->getOperand(2);
      if (VT == MVT::v4i8) {
        MVT WideSrcVT = MVT::v4i16;
        SDValue Packed =
            DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, {Src1, Src2});
        SDValue Res = DAG.getNode(Opc, DL, MVT::v8i8, {Packed, Packed});
        Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      } else {
        MVT WideSrcVT = MVT::v2i32;
        SDValue Packed =
            DAG.getNode(ISD::BUILD_VECTOR, DL, WideSrcVT, {Src1, Src2});
        SDValue Res = DAG.getNode(Opc, DL, MVT::v4i16, {Packed, Packed});
        Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      }
      return;
    }
    case Intrinsic::riscv_psll:
    case Intrinsic::riscv_psrl:
    case Intrinsic::riscv_psra:
    case Intrinsic::riscv_pssha:
    case Intrinsic::riscv_psshar:
    case Intrinsic::riscv_psshl:
    case Intrinsic::riscv_psshlr: {
      MVT VT = N->getSimpleValueType(0);
      if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
        return;
 
      EVT WideVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
      SDValue Op0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT,
                                N->getOperand(1), DAG.getUNDEF(VT));
      SDValue ShAmt = N->getOperand(2);
      ShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, Subtarget.getXLenVT());
      SDValue Res =
          DAG.getNode(getRVPShiftOpcode(IntNo), DL, WideVT, Op0, ShAmt);
      Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
      return;
    }
    case Intrinsic::riscv_pnsrl:
    case Intrinsic::riscv_pnsra:
    case Intrinsic::riscv_pnsrar: {
      MVT VT = N->getSimpleValueType(0);
      if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
        return;
 
      SDValue Src = N->getOperand(1);
      MVT SrcVT = Src.getSimpleValueType();
      if (!((VT == MVT::v4i8 && SrcVT == MVT::v4i16) ||
            (VT == MVT::v2i16 && SrcVT == MVT::v2i32)))
        reportFatalUsageError("unsupported packed narrowing shift intrinsic");
 
      MVT XLenVT = Subtarget.getXLenVT();
      SDValue ShAmt =
          DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(2));
      unsigned Opc;
      switch (IntNo) {
      default:
        llvm_unreachable("Unexpected packed narrowing shift intrinsic");
      case Intrinsic::riscv_pnsrl:
        Opc = RISCVISD::PSRL;
        break;
      case Intrinsic::riscv_pnsra:
        Opc = RISCVISD::PSRA;
        break;
      case Intrinsic::riscv_pnsrar:
        Opc = RISCVISD::PSSHAR;
        ShAmt = DAG.getNode(ISD::AND, DL, XLenVT, ShAmt,
                            DAG.getConstant(31, DL, XLenVT));
        ShAmt = DAG.getNegative(ShAmt, DL, XLenVT);
        break;
      }
 
      SDValue Shifted = DAG.getNode(Opc, DL, SrcVT, Src, ShAmt);
      MVT UnzipVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
      Shifted = DAG.getBitcast(UnzipVT, Shifted);
      SDValue Unzip = DAG.getNode(RISCVISD::PUNZIPE, DL, UnzipVT, Shifted,
                                  DAG.getUNDEF(UnzipVT));
      Results.push_back(DAG.getExtractSubvector(DL, VT, Unzip, 0));
      return;
    }
    case Intrinsic::riscv_predsum:
    case Intrinsic::riscv_predsumu: {
      bool IsSigned = IntNo == Intrinsic::riscv_predsum;
      SDValue Vec = N->getOperand(1);
      MVT VecVT = Vec.getSimpleValueType();
      auto Ext = [&](SDValue V) {
        return DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
                           MVT::i64, V);
      };
      auto RedSum = [&](MVT VT, SDValue V, SDValue Acc) {
        return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, N->getOperand(0), V,
                           Acc);
      };
 
      // RV32: i64 accumulator. Reduce to a 32-bit partial sum, then
      // widening-accumulate into i64 via wadda/waddau (v2i32 uses wadda alone).
      if (!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64) {
        SDValue Acc = N->getOperand(2);
        SDValue Res;
        if (VecVT == MVT::v2i32) {
          Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Acc,
                            Ext(DAG.getExtractVectorElt(DL, MVT::i32, Vec, 0)));
          Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Res,
                            Ext(DAG.getExtractVectorElt(DL, MVT::i32, Vec, 1)));
        } else {
          // The paired predsum.dbs/dhs computes the 32-bit element sum.
          SDValue Partial =
              RedSum(MVT::i32, Vec, DAG.getConstant(0, DL, MVT::i32));
          Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Acc, Ext(Partial));
        }
        Results.push_back(Res);
        return;
      }
 
      // RV64: i32 accumulator. Reduce at i64 (XLEN), then truncate.
      if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
        return;
 
      // Zero the upper lanes (zext.w) so they don't contribute to the sum.
      if (VecVT == MVT::v4i8 || VecVT == MVT::v2i16)
        Vec = widenPackedVectorWithZeros(
            DAG, DL, Vec, VecVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16);
 
      // The result is truncated to i32, so the accumulator's upper bits are
      // unused and need no sign/zero extension.
      SDValue Acc =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
      SDValue Res = RedSum(MVT::i64, Vec, Acc);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_pabdsumu:
    case Intrinsic::riscv_pabdsumau: {
      bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
      // The two packed sources (rs1, rs2) are the last two operands.
      SDValue Rs1 = N->getOperand(N->getNumOperands() - 2);
      SDValue Rs2 = N->getOperand(N->getNumOperands() - 1);
      MVT VecVT = Rs1.getSimpleValueType();
 
      // RV32: i64 result, always from a v8i8 source. The accumulator, if any,
      // folds into the widening add below.
      if (!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64) {
        // Sum of absolute differences of two v4i8 halves.
        auto Sad = [&](SDValue A, SDValue B) {
          SDValue S = DAG.getNode(
              ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
              DAG.getTargetConstant(Intrinsic::riscv_pabdsumu, DL, MVT::i32), A,
              B);
          return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, S);
        };
        auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
        auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
        SDValue Lo = Sad(Rs1Lo, Rs2Lo);
        SDValue Hi = Sad(Rs1Hi, Rs2Hi);
        // (acc + lo) + hi keeps the accumulate chained so it folds into a
        // single waddau; without an accumulator lo + hi folds into waddu.
        SDValue Res =
            IsAcc ? DAG.getNode(ISD::ADD, DL, MVT::i64, N->getOperand(1), Lo)
                  : Lo;
        Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Res, Hi);
        Results.push_back(Res);
        return;
      }
 
      // RV64: i32 result, so reduce at i64 and truncate. The source is v4i8 or
      // v8i8; widen a v4i8 to v8i8, zeroing the upper bytes (v8i8 is legal).
      if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
        return;
      if (VecVT == MVT::v4i8) {
        Rs1 = widenPackedVectorWithZeros(DAG, DL, Rs1, MVT::v8i8);
        Rs2 = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v8i8);
      }
      SmallVector<SDValue, 4> Ops = {N->getOperand(0)};
      if (IsAcc)
        Ops.push_back(
            DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)));
      Ops.push_back(Rs1);
      Ops.push_back(Rs2);
      SDValue Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, Ops);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_mqacc_00:
    case Intrinsic::riscv_mqacc_01:
    case Intrinsic::riscv_mqacc_11:
    case Intrinsic::riscv_mqracc_00:
    case Intrinsic::riscv_mqracc_01:
    case Intrinsic::riscv_mqracc_11: {
      EVT VT = N->getValueType(0);
      if (Subtarget.is64Bit() && VT == MVT::i32) {
        SDValue Rd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
                                 N->getOperand(1));
        auto WidenSrc = [&](SDValue V) {
          return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
                             {V, DAG.getUNDEF(MVT::v2i16)});
        };
        SDValue Rs1 = WidenSrc(N->getOperand(2));
        SDValue Rs2 = WidenSrc(N->getOperand(3));
        unsigned Opc = getRVPQFormatAccOpcode(IntNo);
        SDValue Res = DAG.getNode(Opc, DL, MVT::v2i32, Rd, Rs1, Rs2);
        Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
        return;
      }
 
      if (!Subtarget.is64Bit() && VT == MVT::i64) {
        bool IsRound = IntNo == Intrinsic::riscv_mqracc_00 ||
                       IntNo == Intrinsic::riscv_mqracc_01 ||
                       IntNo == Intrinsic::riscv_mqracc_11;
        bool Hi1 = IntNo == Intrinsic::riscv_mqacc_11 ||
                   IntNo == Intrinsic::riscv_mqracc_11;
        bool Hi2 = IntNo == Intrinsic::riscv_mqacc_01 ||
                   IntNo == Intrinsic::riscv_mqacc_11 ||
                   IntNo == Intrinsic::riscv_mqracc_01 ||
                   IntNo == Intrinsic::riscv_mqracc_11;
        MVT XLenVT = Subtarget.getXLenVT();
        SDValue Rs1 = N->getOperand(2);
        SDValue Rs2 = N->getOperand(3);
        SDValue A = DAG.getExtractVectorElt(DL, XLenVT, Rs1, Hi1 ? 1 : 0);
        SDValue B = DAG.getExtractVectorElt(DL, XLenVT, Rs2, Hi2 ? 1 : 0);
        auto [RdLo, RdHi] =
            DAG.SplitScalar(N->getOperand(1), DL, XLenVT, XLenVT);
        unsigned Opc = IsRound ? RISCVISD::MQRWACC : RISCVISD::MQWACC;
        SDVTList VTs = DAG.getVTList(XLenVT, XLenVT);
        SDValue Acc = DAG.getNode(Opc, DL, VTs, {RdLo, RdHi, A, B});
        Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
                                      Acc.getValue(0), Acc.getValue(1)));
        return;
      }
      return;
    }
    case Intrinsic::riscv_orc_b:
    case Intrinsic::riscv_brev8:
    case Intrinsic::riscv_sha256sig0:
    case Intrinsic::riscv_sha256sig1:
    case Intrinsic::riscv_sha256sum0:
    case Intrinsic::riscv_sha256sum1:
    case Intrinsic::riscv_sm3p0:
    case Intrinsic::riscv_sm3p1: {
      if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
        return;
      unsigned Opc;
      switch (IntNo) {
      case Intrinsic::riscv_orc_b:      Opc = RISCVISD::ORC_B;      break;
      case Intrinsic::riscv_brev8:      Opc = RISCVISD::BREV8;      break;
      case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
      case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
      case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
      case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
      case Intrinsic::riscv_sm3p0:      Opc = RISCVISD::SM3P0;      break;
      case Intrinsic::riscv_sm3p1:      Opc = RISCVISD::SM3P1;      break;
      }
 
      SDValue NewOp =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
      SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp);
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_sm4ks:
    case Intrinsic::riscv_sm4ed: {
      unsigned Opc =
          IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
      SDValue NewOp0 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
      SDValue NewOp1 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
      SDValue Res =
          DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, N->getOperand(3));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_mopr: {
      if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
        return;
      SDValue NewOp =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
      SDValue Res = DAG.getNode(
          RISCVISD::MOP_R, DL, MVT::i64, NewOp,
          DAG.getTargetConstant(N->getConstantOperandVal(2), DL, MVT::i64));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_moprr: {
      if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
        return;
      SDValue NewOp0 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
      SDValue NewOp1 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
      SDValue Res = DAG.getNode(
          RISCVISD::MOP_RR, DL, MVT::i64, NewOp0, NewOp1,
          DAG.getTargetConstant(N->getConstantOperandVal(3), DL, MVT::i64));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_clmulh:
    case Intrinsic::riscv_clmulr: {
      if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
        return;
 
      // Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
      // to the full 128-bit clmul result of multiplying two xlen values.
      // Perform clmulr or clmulh on the shifted values. Finally, extract the
      // upper 32 bits.
      //
      // The alternative is to mask the inputs to 32 bits and use clmul, but
      // that requires two shifts to mask each input without zext.w.
      // FIXME: If the inputs are known zero extended or could be freely
      // zero extended, the mask form would be better.
      SDValue NewOp0 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
      SDValue NewOp1 =
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
      NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
                           DAG.getConstant(32, DL, MVT::i64));
      NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
                           DAG.getConstant(32, DL, MVT::i64));
      unsigned Opc =
          IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
      SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1);
      Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
                        DAG.getConstant(32, DL, MVT::i64));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
      return;
    }
    case Intrinsic::riscv_vmv_x_s: {
      EVT VT = N->getValueType(0);
      MVT XLenVT = Subtarget.getXLenVT();
      if (VT.bitsLT(XLenVT)) {
        // Simple case just extract using vmv.x.s and truncate.
        SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
                                      Subtarget.getXLenVT(), N->getOperand(1));
        Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
        return;
      }
 
      assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
             "Unexpected custom legalization");
 
      // We need to do the move in two steps.
      SDValue Vec = N->getOperand(1);
      MVT VecVT = Vec.getSimpleValueType();
 
      // First extract the lower XLEN bits of the element.
      SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
 
      // To extract the upper XLEN bits of the vector element, shift the first
      // element right by 32 bits and re-extract the lower XLEN bits.
      auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget);
 
      SDValue ThirtyTwoV =
          DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
                      DAG.getConstant(32, DL, XLenVT), VL);
      SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
                                   DAG.getUNDEF(VecVT), Mask, VL);
      SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
 
      Results.push_back(
          DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
      break;
    }
    }
    break;
  }
  case ISD::VECREDUCE_ADD:
  case ISD::VECREDUCE_AND:
  case ISD::VECREDUCE_OR:
  case ISD::VECREDUCE_XOR:
  case ISD::VECREDUCE_SMAX:
  case ISD::VECREDUCE_UMAX:
  case ISD::VECREDUCE_SMIN:
  case ISD::VECREDUCE_UMIN:
    if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
      Results.push_back(V);
    break;
  case ISD::VP_REDUCE_ADD:
  case ISD::VP_REDUCE_AND:
  case ISD::VP_REDUCE_OR:
  case ISD::VP_REDUCE_XOR:
  case ISD::VP_REDUCE_SMAX:
  case ISD::VP_REDUCE_UMAX:
  case ISD::VP_REDUCE_SMIN:
  case ISD::VP_REDUCE_UMIN:
    if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
      Results.push_back(V);
    break;
  case ISD::GET_ROUNDING: {
    SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
    SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0));
    Results.push_back(Res.getValue(0));
    Results.push_back(Res.getValue(1));
    break;
  }
  }
}
 
/// Given a binary operator, return the *associative* generic ISD::VECREDUCE_OP
/// which corresponds to it.
static unsigned getVecReduceOpcode(unsigned Opc) {
  switch (Opc) {
  default:
    llvm_unreachable("Unhandled binary to transform reduction");
  case ISD::ADD:
    return ISD::VECREDUCE_ADD;
  case ISD::UMAX:
    return ISD::VECREDUCE_UMAX;
  case ISD::SMAX:
    return ISD::VECREDUCE_SMAX;
  case ISD::UMIN:
    return ISD::VECREDUCE_UMIN;
  case ISD::SMIN:
    return ISD::VECREDUCE_SMIN;
  case ISD::AND:
    return ISD::VECREDUCE_AND;
  case ISD::OR:
    return ISD::VECREDUCE_OR;
  case ISD::XOR:
    return ISD::VECREDUCE_XOR;
  case ISD::FADD:
    // Note: This is the associative form of the generic reduction opcode.
    return ISD::VECREDUCE_FADD;
  case ISD::FMAXNUM:
    return ISD::VECREDUCE_FMAX;
  case ISD::FMINNUM:
    return ISD::VECREDUCE_FMIN;
  }
}
 
/// Perform two related transforms whose purpose is to incrementally recognize
/// an explode_vector followed by scalar reduction as a vector reduction node.
/// This exists to recover from a deficiency in SLP which can't handle
/// forests with multiple roots sharing common nodes.  In some cases, one
/// of the trees will be vectorized, and the other will remain (unprofitably)
/// scalarized.
static SDValue
combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
 
  // This transforms need to run before all integer types have been legalized
  // to i64 (so that the vector element type matches the add type), and while
  // it's safe to introduce odd sized vector types.
  if (DAG.NewNodesMustHaveLegalTypes)
    return SDValue();
 
  // Without V, this transform isn't useful.  We could form the (illegal)
  // operations and let them be scalarized again, but there's really no point.
  if (!Subtarget.hasVInstructions())
    return SDValue();
 
  const SDLoc DL(N);
  const EVT VT = N->getValueType(0);
  const unsigned Opc = N->getOpcode();
 
  if (!VT.isInteger()) {
    switch (Opc) {
    default:
      return SDValue();
    case ISD::FADD:
      // For FADD, we only handle the case with reassociation allowed.  We
      // could handle strict reduction order, but at the moment, there's no
      // known reason to, and the complexity isn't worth it.
      if (!N->getFlags().hasAllowReassociation())
        return SDValue();
      break;
    case ISD::FMAXNUM:
    case ISD::FMINNUM:
      break;
    }
  }
 
  const unsigned ReduceOpc = getVecReduceOpcode(Opc);
  assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
         "Inconsistent mappings");
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
 
  if (!LHS.hasOneUse() || !RHS.hasOneUse())
    return SDValue();
 
  if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    std::swap(LHS, RHS);
 
  if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
      !isa<ConstantSDNode>(RHS.getOperand(1)))
    return SDValue();
 
  uint64_t RHSIdx = cast<ConstantSDNode>(RHS.getOperand(1))->getLimitedValue();
  SDValue SrcVec = RHS.getOperand(0);
  EVT SrcVecVT = SrcVec.getValueType();
  if (SrcVecVT.getVectorElementType() != VT)
    return SDValue();
  if (SrcVecVT.isScalableVector())
    return SDValue();
 
  if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen())
    return SDValue();
 
  // match binop (extract_vector_elt V, 0), (extract_vector_elt V, 1) to
  // reduce_op (extract_subvector [2 x VT] from V).  This will form the
  // root of our reduction tree. TODO: We could extend this to any two
  // adjacent aligned constant indices if desired.
  if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
      LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(LHS.getOperand(1))) {
    uint64_t LHSIdx =
      cast<ConstantSDNode>(LHS.getOperand(1))->getLimitedValue();
    if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
      EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, 2);
      SDValue Vec = DAG.getExtractSubvector(DL, ReduceVT, SrcVec, 0);
      return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags());
    }
  }
 
  // Match (binop (reduce (extract_subvector V, 0),
  //                      (extract_vector_elt V, sizeof(SubVec))))
  // into a reduction of one more element from the original vector V.
  if (LHS.getOpcode() != ReduceOpc)
    return SDValue();
 
  SDValue ReduceVec = LHS.getOperand(0);
  if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
      ReduceVec.hasOneUse() && ReduceVec.getOperand(0) == RHS.getOperand(0) &&
      isNullConstant(ReduceVec.getOperand(1)) &&
      ReduceVec.getValueType().getVectorNumElements() == RHSIdx) {
    // For illegal types (e.g. 3xi32), most will be combined again into a
    // wider (hopefully legal) type.  If this is a terminal state, we are
    // relying on type legalization here to produce something reasonable
    // and this lowering quality could probably be improved. (TODO)
    EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, RHSIdx + 1);
    SDValue Vec = DAG.getExtractSubvector(DL, ReduceVT, SrcVec, 0);
    return DAG.getNode(ReduceOpc, DL, VT, Vec,
                       ReduceVec->getFlags() & N->getFlags());
  }
 
  return SDValue();
}
 
 
// Try to fold (<bop> x, (reduction.<bop> vec, start))
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
  auto BinOpToRVVReduce = [](unsigned Opc) {
    switch (Opc) {
    default:
      llvm_unreachable("Unhandled binary to transform reduction");
    case ISD::ADD:
      return RISCVISD::VECREDUCE_ADD_VL;
    case ISD::UMAX:
      return RISCVISD::VECREDUCE_UMAX_VL;
    case ISD::SMAX:
      return RISCVISD::VECREDUCE_SMAX_VL;
    case ISD::UMIN:
      return RISCVISD::VECREDUCE_UMIN_VL;
    case ISD::SMIN:
      return RISCVISD::VECREDUCE_SMIN_VL;
    case ISD::AND:
      return RISCVISD::VECREDUCE_AND_VL;
    case ISD::OR:
      return RISCVISD::VECREDUCE_OR_VL;
    case ISD::XOR:
      return RISCVISD::VECREDUCE_XOR_VL;
    case ISD::FADD:
      return RISCVISD::VECREDUCE_FADD_VL;
    case ISD::FMAXNUM:
      return RISCVISD::VECREDUCE_FMAX_VL;
    case ISD::FMINNUM:
      return RISCVISD::VECREDUCE_FMIN_VL;
    }
  };
 
  auto IsReduction = [&BinOpToRVVReduce](SDValue V, unsigned Opc) {
    return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
           isNullConstant(V.getOperand(1)) &&
           V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
  };
 
  unsigned Opc = N->getOpcode();
  unsigned ReduceIdx;
  if (IsReduction(N->getOperand(0), Opc))
    ReduceIdx = 0;
  else if (IsReduction(N->getOperand(1), Opc))
    ReduceIdx = 1;
  else
    return SDValue();
 
  // Skip if FADD disallows reassociation but the combiner needs.
  if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
    return SDValue();
 
  SDValue Extract = N->getOperand(ReduceIdx);
  SDValue Reduce = Extract.getOperand(0);
  if (!Extract.hasOneUse() || !Reduce.hasOneUse())
    return SDValue();
 
  SDValue ScalarV = Reduce.getOperand(2);
  EVT ScalarVT = ScalarV.getValueType();
  if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
      ScalarV.getOperand(0)->isUndef() &&
      isNullConstant(ScalarV.getOperand(2)))
    ScalarV = ScalarV.getOperand(1);
 
  // Make sure that ScalarV is a splat with VL=1.
  if (ScalarV.getOpcode() != RISCVISD::VFMV_S_F_VL &&
      ScalarV.getOpcode() != RISCVISD::VMV_S_X_VL &&
      ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
    return SDValue();
 
  if (!isNonZeroAVL(ScalarV.getOperand(2)))
    return SDValue();
 
  // Check the scalar of ScalarV is neutral element
  // TODO: Deal with value other than neutral element.
  if (!DAG.isIdentityElement(N->getOpcode(), N->getFlags(),
                             ScalarV.getOperand(1), 0))
    return SDValue();
 
  // If the AVL is zero, operand 0 will be returned. So it's not safe to fold.
  // FIXME: We might be able to improve this if operand 0 is undef.
  if (!isNonZeroAVL(Reduce.getOperand(5)))
    return SDValue();
 
  SDValue NewStart = N->getOperand(1 - ReduceIdx);
 
  SDLoc DL(N);
  SDValue NewScalarV =
      lowerScalarInsert(NewStart, ScalarV.getOperand(2),
                        ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
 
  // If we looked through an INSERT_SUBVECTOR we need to restore it.
  if (ScalarVT != ScalarV.getValueType())
    NewScalarV =
        DAG.getInsertSubvector(DL, DAG.getUNDEF(ScalarVT), NewScalarV, 0);
 
  SDValue Ops[] = {Reduce.getOperand(0), Reduce.getOperand(1),
                   NewScalarV,           Reduce.getOperand(3),
                   Reduce.getOperand(4), Reduce.getOperand(5)};
  SDValue NewReduce =
      DAG.getNode(Reduce.getOpcode(), DL, Reduce.getValueType(), Ops);
  return DAG.getNode(Extract.getOpcode(), DL, Extract.getValueType(), NewReduce,
                     Extract.getOperand(1));
}
 
// Optimize (add (shl x, c0), (shl y, c1)) ->
//          (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
// or
//          (SLLI (QC.SHLADD x, y, c1 - c0), c0), if 4 <= (c1-c0) <=31.
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  // Perform this optimization only in the zba/xandesperf/xqciac/xtheadba
  // extension.
  if (!Subtarget.hasShlAdd(3))
    return SDValue();
 
  // Skip for vector types and larger types.
  EVT VT = N->getValueType(0);
  if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
    return SDValue();
 
  // The two operand nodes must be SHL and have no other use.
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
      !N0->hasOneUse() || !N1->hasOneUse())
    return SDValue();
 
  // Check c0 and c1.
  auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
  auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
  if (!N0C || !N1C)
    return SDValue();
  int64_t C0 = N0C->getSExtValue();
  int64_t C1 = N1C->getSExtValue();
  if (C0 <= 0 || C1 <= 0)
    return SDValue();
 
  int64_t Diff = std::abs(C0 - C1);
  if (!Subtarget.hasShlAdd(Diff))
    return SDValue();
 
  // Build nodes.
  SDLoc DL(N);
  int64_t Bits = std::min(C0, C1);
  SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
  SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
  SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, NL,
                              DAG.getTargetConstant(Diff, DL, VT), NS);
  return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
}
 
// Check if this SDValue is an add immediate that is fed by a shift of 1, 2,
// or 3.
static SDValue combineShlAddIAddImpl(SDNode *N, SDValue AddI, SDValue Other,
                                     SelectionDAG &DAG) {
  using namespace llvm::SDPatternMatch;
 
  // Looking for a reg-reg add and not an addi.
  if (isa<ConstantSDNode>(N->getOperand(1)))
    return SDValue();
 
  // Based on testing it seems that performance degrades if the ADDI has
  // more than 2 uses.
  if (AddI->use_size() > 2)
    return SDValue();
 
  APInt AddVal;
  SDValue SHLVal;
  if (!sd_match(AddI, m_Add(m_Value(SHLVal), m_ConstInt(AddVal))))
    return SDValue();
 
  APInt VShift;
  if (!sd_match(SHLVal, m_OneUse(m_Shl(m_Value(), m_ConstInt(VShift)))))
    return SDValue();
 
  if (VShift.slt(1) || VShift.sgt(3))
    return SDValue();
 
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  // The shift must be positive but the add can be signed.
  uint64_t ShlConst = VShift.getZExtValue();
  int64_t AddConst = AddVal.getSExtValue();
 
  SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, SHLVal->getOperand(0),
                              DAG.getTargetConstant(ShlConst, DL, VT), Other);
  return DAG.getNode(ISD::ADD, DL, VT, SHADD,
                     DAG.getSignedConstant(AddConst, DL, VT));
}
 
// Optimize (add (add (shl x, c0),  c1), y) ->
//          (ADDI (SH*ADD y, x), c1), if c0 equals to [1|2|3].
static SDValue combineShlAddIAdd(SDNode *N, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  // Perform this optimization only in the zba extension.
  if (!ReassocShlAddiAdd || !Subtarget.hasShlAdd(3))
    return SDValue();
 
  // Skip for vector types and larger types.
  EVT VT = N->getValueType(0);
  if (VT != Subtarget.getXLenVT())
    return SDValue();
 
  SDValue AddI = N->getOperand(0);
  SDValue Other = N->getOperand(1);
  if (SDValue V = combineShlAddIAddImpl(N, AddI, Other, DAG))
    return V;
  if (SDValue V = combineShlAddIAddImpl(N, Other, AddI, DAG))
    return V;
  return SDValue();
}
 
// Combine a constant select operand into its use:
//
// (and (select cond, -1, c), x)
//   -> (select cond, x, (and x, c))  [AllOnes=1]
// (or  (select cond, 0, c), x)
//   -> (select cond, x, (or x, c))  [AllOnes=0]
// (xor (select cond, 0, c), x)
//   -> (select cond, x, (xor x, c))  [AllOnes=0]
// (add (select cond, 0, c), x)
//   -> (select cond, x, (add x, c))  [AllOnes=0]
// (sub x, (select cond, 0, c))
//   -> (select cond, x, (sub x, c))  [AllOnes=0]
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
                                   SelectionDAG &DAG, bool AllOnes,
                                   const RISCVSubtarget &Subtarget) {
  EVT VT = N->getValueType(0);
 
  // Skip vectors.
  if (VT.isVector())
    return SDValue();
 
  if (!Subtarget.hasConditionalMoveFusion()) {
    // (select cond, x, (and x, c)) has custom lowering with Zicond.
    if (!Subtarget.hasStdExtZicond() || N->getOpcode() != ISD::AND)
      return SDValue();
 
    // Maybe harmful when condition code has multiple use.
    if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(0).hasOneUse())
      return SDValue();
 
    // Maybe harmful when VT is wider than XLen.
    if (VT.getSizeInBits() > Subtarget.getXLen())
      return SDValue();
  }
 
  if ((Slct.getOpcode() != ISD::SELECT &&
       Slct.getOpcode() != RISCVISD::SELECT_CC) ||
      !Slct.hasOneUse())
    return SDValue();
 
  auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
    return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
  };
 
  bool SwapSelectOps;
  unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
  SDValue TrueVal = Slct.getOperand(1 + OpOffset);
  SDValue FalseVal = Slct.getOperand(2 + OpOffset);
  SDValue NonConstantVal;
  if (isZeroOrAllOnes(TrueVal, AllOnes)) {
    SwapSelectOps = false;
    NonConstantVal = FalseVal;
  } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
    SwapSelectOps = true;
    NonConstantVal = TrueVal;
  } else
    return SDValue();
 
  // Slct is now know to be the desired identity constant when CC is true.
  TrueVal = OtherOp;
  FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
  // Unless SwapSelectOps says the condition should be false.
  if (SwapSelectOps)
    std::swap(TrueVal, FalseVal);
 
  if (Slct.getOpcode() == RISCVISD::SELECT_CC)
    return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
                       {Slct.getOperand(0), Slct.getOperand(1),
                        Slct.getOperand(2), TrueVal, FalseVal});
 
  return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
                     {Slct.getOperand(0), TrueVal, FalseVal});
}
 
// Attempt combineSelectAndUse on each operand of a commutative operator N.
static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
                                              bool AllOnes,
                                              const RISCVSubtarget &Subtarget) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes, Subtarget))
    return Result;
  if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes, Subtarget))
    return Result;
  return SDValue();
}
 
// Transform (add (mul x, c0), c1) ->
//           (add (mul (add x, c1/c0), c0), c1%c0).
// if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
// that should be excluded is when c0*(c1/c0) is simm12, which will lead
// to an infinite loop in DAGCombine if transformed.
// Or transform (add (mul x, c0), c1) ->
//              (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
// if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
// case that should be excluded is when c0*(c1/c0+1) is simm12, which will
// lead to an infinite loop in DAGCombine if transformed.
// Or transform (add (mul x, c0), c1) ->
//              (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
// if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
// case that should be excluded is when c0*(c1/c0-1) is simm12, which will
// lead to an infinite loop in DAGCombine if transformed.
// Or transform (add (mul x, c0), c1) ->
//              (mul (add x, c1/c0), c0).
// if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
                                     const RISCVSubtarget &Subtarget) {
  // Skip for vector types and larger types.
  EVT VT = N->getValueType(0);
  if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
    return SDValue();
  // The first operand node must be a MUL and has no other use.
  SDValue N0 = N->getOperand(0);
  if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
    return SDValue();
  // Check if c0 and c1 match above conditions.
  auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
  auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!N0C || !N1C)
    return SDValue();
  // If N0C has multiple uses it's possible one of the cases in
  // DAGCombiner::isMulAddWithConstProfitable will be true, which would result
  // in an infinite loop.
  if (!N0C->hasOneUse())
    return SDValue();
  int64_t C0 = N0C->getSExtValue();
  int64_t C1 = N1C->getSExtValue();
  int64_t CA, CB;
  // If C1 already fits in an add immediate, there is nothing to split out: the
  // (add (mul x, c0), c1) form is already canonical/cheap. Splitting it would
  // fight the generic DAGCombiner fold add(mul(add(A, CA), CM), CB) ->
  // add(mul(A, CM), CM*CA+CB) (which is gated on isLegalAddImmediate) and cause
  // an infinite loop.
  if (C0 == -1 || C0 == 0 || C0 == 1 ||
      Subtarget.getTargetLowering()->isLegalAddImmediate(C1))
    return SDValue();
  // Search for proper CA (non-zero) and CB that both are simm12.
  if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
      !isInt<12>(C0 * (C1 / C0))) {
    CA = C1 / C0;
    CB = C1 % C0;
  } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
             isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
    CA = C1 / C0 + 1;
    CB = C1 % C0 - C0;
  } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
             isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
    CA = C1 / C0 - 1;
    CB = C1 % C0 + C0;
  } else
    return SDValue();
  // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
  SDLoc DL(N);
  SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
                             DAG.getSignedConstant(CA, DL, VT));
  SDValue New1 =
      DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getSignedConstant(C0, DL, VT));
  return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getSignedConstant(CB, DL, VT));
}
 
// add (zext, zext) -> zext (add (zext, zext))
// sub (zext, zext) -> sext (sub (zext, zext))
// mul (zext, zext) -> zext (mul (zext, zext))
// sdiv (zext, zext) -> zext (sdiv (zext, zext))
// udiv (zext, zext) -> zext (udiv (zext, zext))
// srem (zext, zext) -> zext (srem (zext, zext))
// urem (zext, zext) -> zext (urem (zext, zext))
//
// where the sum of the extend widths match, and the the range of the bin op
// fits inside the width of the narrower bin op. (For profitability on rvv, we
// use a power of two for both inner and outer extend.)
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
 
  EVT VT = N->getValueType(0);
  if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
    return SDValue();
  if (!N0.hasOneUse() || !N1.hasOneUse())
    return SDValue();
 
  SDValue Src0 = N0.getOperand(0);
  SDValue Src1 = N1.getOperand(0);
  EVT SrcVT = Src0.getValueType();
  if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) ||
      SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 ||
      SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
    return SDValue();
 
  LLVMContext &C = *DAG.getContext();
  EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C);
  EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount());
 
  Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0);
  Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1);
 
  // Src0 and Src1 are zero extended, so they're always positive if signed.
  //
  // sub can produce a negative from two positive operands, so it needs sign
  // extended. Other nodes produce a positive from two positive operands, so
  // zero extend instead.
  unsigned OuterExtend =
      N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
 
  return DAG.getNode(
      OuterExtend, SDLoc(N), VT,
      DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1));
}
 
// Try to turn (add (xor bool, 1) -1) into (neg bool).
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // RHS should be -1.
  if (!isAllOnesConstant(N1))
    return SDValue();
 
  // Look for (xor X, 1).
  if (N0.getOpcode() != ISD::XOR || !isOneConstant(N0.getOperand(1)))
    return SDValue();
 
  // First xor input should be 0 or 1.
  APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
  if (!DAG.MaskedValueIsZero(N0.getOperand(0), Mask))
    return SDValue();
 
  // Emit a negate of the setcc.
  return DAG.getNegative(N0.getOperand(0), DL, VT);
}
 
// Fold (add X, (mulhs X, C)) -> (mulhsu X, C) if C is negative. This occurs
// in the expansion of sdiv i32 X, 7 using magic multiply.
//
// mulhs returns the hi from X * C = hi * 2^32 + lo.
//
// Since C<0, u(C) as an unsigned constant is 2^32 + C = u(C).
// mulhsu computes
//  X * u(C0) = X * (C + 2^32)
//            = X * 2^32 + C * X // C * X is the same as mulhs
//            = X * 2^32 + hi * 2^32 + lo
//            = (X + hi) * 2^32 + lo
// So mulhsu computes (X + hi).
static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  EVT VT = N->getValueType(0);
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  bool IsPExtPackedDoubleType =
      VT.isSimple() && Subtarget.isPExtPackedDoubleType(VT.getSimpleVT());
  if (!TLI.isOperationLegal(ISD::MULHS, VT) && !IsPExtPackedDoubleType &&
      !(Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8) &&
      !(Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::v8i8))
    return SDValue();
 
  using namespace SDPatternMatch;
  SDValue X, Mulh;
  APInt C;
  if (!sd_match(N,
                m_Add(m_Value(X),
                      m_OneUse(m_Value(Mulh, m_BinOp(ISD::MULHS, m_Deferred(X),
                                                     m_ConstInt(C)))))) ||
      !C.isNegative())
    return SDValue();
 
  SDLoc DL(N);
 
  // We don't have a v4i8 MULHSU instruction, use a WMULSU+SRL+TRUNC.
  auto MakePWMulSU = [&](SDValue A, SDValue B) -> SDValue {
    SDValue WMul = DAG.getNode(RISCVISD::PWMULSU, DL, MVT::v4i16, A, B);
    SDValue Shifted = DAG.getNode(ISD::SRL, DL, MVT::v4i16, WMul,
                                  DAG.getConstant(8, DL, MVT::v4i16));
    return DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i8, Shifted);
  };
 
  // We need to split double wide vectors ourselves, op legalization won't
  // run for custom nodes.
  if (IsPExtPackedDoubleType) {
    MVT HalfVT = VT.getSimpleVT().getHalfNumVectorElementsVT();
    auto [XLo, XHi] = DAG.SplitVector(X, DL, HalfVT, HalfVT);
    auto [CLo, CHi] = DAG.SplitVector(Mulh.getOperand(1), DL, HalfVT, HalfVT);
    SDValue ResLo, ResHi;
    if (HalfVT == MVT::v4i8) {
      ResLo = MakePWMulSU(XLo, CLo);
      ResHi = MakePWMulSU(XHi, CHi);
    } else {
      ResLo = DAG.getNode(RISCVISD::MULHSU, DL, HalfVT, XLo, CLo);
      ResHi = DAG.getNode(RISCVISD::MULHSU, DL, HalfVT, XHi, CHi);
    }
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
  }
 
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8)
    return MakePWMulSU(X, Mulh.getOperand(1));
 
  // We don't have a v8i8 MULHSU instruction on RV64 either; build it from a
  // pair of widening byte multiplies recombined with PPAIRO.
  if (Subtarget.hasStdExtP() && Subtarget.is64Bit() && VT == MVT::v8i8) {
    SDValue C = Mulh.getOperand(1);
    SDValue Lo = DAG.getNode(RISCVISD::PMULSU_HALVES_00, DL, MVT::v4i16, X, C);
    SDValue Hi = DAG.getNode(RISCVISD::PMULSU_HALVES_11, DL, MVT::v4i16, X, C);
    return DAG.getNode(RISCVISD::PPAIRO, DL, VT, DAG.getBitcast(VT, Lo),
                       DAG.getBitcast(VT, Hi));
  }
 
  return DAG.getNode(RISCVISD::MULHSU, DL, VT, X, Mulh.getOperand(1));
}
 
// Fold an add of a multiply-parts product into the accumulating form.
static SDValue combineAddMulParts(SDNode *N, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  if (!Subtarget.hasStdExtP())
    return SDValue();
 
  for (unsigned I = 0; I != 2; ++I) {
    SDValue Mul = N->getOperand(I);
    if (Mul.getOpcode() != ISD::INTRINSIC_WO_CHAIN || !Mul.hasOneUse())
      continue;
    Intrinsic::ID AccId =
        getRVPMulPartsAccIntrinsic(Mul.getConstantOperandVal(0));
    if (AccId == Intrinsic::not_intrinsic)
      continue;
 
    SDLoc DL(N);
    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
                       DAG.getTargetConstant(AccId, DL, Subtarget.getXLenVT()),
                       N->getOperand(1 - I), Mul.getOperand(1),
                       Mul.getOperand(2));
  }
  return SDValue();
}
 
static SDValue combinePExtWideningAddSub(SDNode *N, SelectionDAG &DAG,
                                         const RISCVSubtarget &Subtarget) {
  // Recognize the RV64 decompositions listed for the 32-bit packed widening
  // add/sub operations. Mixed signed/unsigned PM2 forms are outside that group.
  if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit())
    return SDValue();
 
  unsigned Opcode = N->getOpcode();
  if (Opcode != ISD::ADD && Opcode != ISD::SUB)
    return SDValue();
 
  EVT VT = N->getValueType(0);
  if (VT != MVT::v4i16 && VT != MVT::v2i32)
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  unsigned ExtendOpcode = N0.getOpcode();
  if ((ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND) ||
      N1.getOpcode() != ExtendOpcode || !N0.hasOneUse() || !N1.hasOneUse())
    return SDValue();
  bool IsSExt = ExtendOpcode == ISD::SIGN_EXTEND;
 
  SDValue A = N0.getOperand(0);
  SDValue B = N1.getOperand(0);
  MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
  MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
  if (A.getValueType() != SrcVT || B.getValueType() != SrcVT)
    return SDValue();
 
  if (VT == MVT::v4i16 && !IsSExt)
    return SDValue();
 
  bool IsPM2Halfword = VT == MVT::v2i32 && (IsSExt || Opcode == ISD::ADD);
  if (VT == MVT::v2i32 && !IsPM2Halfword)
    return SDValue();
 
  SDLoc DL(N);
  A = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, A, DAG.getUNDEF(SrcVT));
  B = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, B, DAG.getUNDEF(SrcVT));
 
  SDValue Zip = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, A, B);
  if (VT == MVT::v4i16) {
    SDValue ZipAsVT = DAG.getBitcast(VT, Zip);
    SDValue Low = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, ZipAsVT,
                              DAG.getValueType(MVT::v4i8));
    SDValue High = DAG.getNode(RISCVISD::PSRA, DL, VT, ZipAsVT,
                               DAG.getConstant(8, DL, MVT::i64));
    return DAG.getNode(Opcode, DL, VT, Low, High);
  }
 
  SDValue Ones = DAG.getConstant(1, DL, LegalSrcVT);
  if (IsSExt) {
    unsigned Opc = Opcode == ISD::ADD ? RISCVISD::PM2ADD_H : RISCVISD::PM2SUB_H;
    return DAG.getNode(Opc, DL, VT, Zip, Ones);
  }
 
  if (Opcode == ISD::ADD)
    return DAG.getNode(RISCVISD::PM2ADDU_H, DL, VT, Zip, Ones);
 
  return SDValue();
}
 
static SDValue combinePExtWideningAddAcc(SDNode *N, SelectionDAG &DAG,
                                         const RISCVSubtarget &Subtarget) {
  using namespace SDPatternMatch;
 
  if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit())
    return SDValue();
 
  if (N->getOpcode() != ISD::ADD)
    return SDValue();
 
  MVT VT = N->getSimpleValueType(0);
  if (VT != MVT::v4i16 && VT != MVT::v2i32)
    return SDValue();
 
  auto MatchExtend = [](SDValue V, unsigned ExtendOpcode, MVT SrcVT,
                        SDValue &Src) {
    return sd_match(
        V, m_OneUse(m_Node(ExtendOpcode, m_Value(Src, m_SpecificVT(SrcVT)))));
  };
 
  auto Match = [&](SDValue Ext, SDValue Add, SDValue &Acc, SDValue &A,
                   SDValue &B, bool &IsSExt) {
    MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
    unsigned ExtendOpcode = Ext.getOpcode();
    if (ExtendOpcode != ISD::SIGN_EXTEND && ExtendOpcode != ISD::ZERO_EXTEND)
      return false;
 
    if (!MatchExtend(Ext, ExtendOpcode, SrcVT, B))
      return false;
 
    if (!sd_match(Add, m_OneUse(m_Add(m_Value(), m_Value()))))
      return false;
 
    if (MatchExtend(Add.getOperand(0), ExtendOpcode, SrcVT, A))
      Acc = Add.getOperand(1);
    else if (MatchExtend(Add.getOperand(1), ExtendOpcode, SrcVT, A))
      Acc = Add.getOperand(0);
    else
      return false;
 
    if (Acc.getValueType() != VT)
      return false;
 
    IsSExt = ExtendOpcode == ISD::SIGN_EXTEND;
    return true;
  };
 
  SDValue Acc, A, B;
  bool IsSExt;
  if (!Match(N->getOperand(0), N->getOperand(1), Acc, A, B, IsSExt) &&
      !Match(N->getOperand(1), N->getOperand(0), Acc, A, B, IsSExt))
    return SDValue();
 
  if (VT == MVT::v4i16 && !IsSExt)
    return SDValue();
 
  SDLoc DL(N);
  MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
  MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
  A = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, A, DAG.getUNDEF(SrcVT));
  B = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, B, DAG.getUNDEF(SrcVT));
 
  SDValue Zip = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, A, B);
  if (VT == MVT::v4i16) {
    SDValue ZipAsVT = DAG.getBitcast(VT, Zip);
    SDValue Low = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, ZipAsVT,
                              DAG.getValueType(MVT::v4i8));
    SDValue High = DAG.getNode(RISCVISD::PSRA, DL, VT, ZipAsVT,
                               DAG.getConstant(8, DL, MVT::i64));
    return DAG.getNode(ISD::ADD, DL, VT,
                       DAG.getNode(ISD::ADD, DL, VT, Acc, Low), High);
  }
 
  SDValue Ones = DAG.getConstant(1, DL, LegalSrcVT);
  unsigned Opc = IsSExt ? RISCVISD::PM2ADDA_H : RISCVISD::PM2ADDAU_H;
  return DAG.getNode(Opc, DL, VT, Acc, Zip, Ones);
}
 
// Fold sub(add(Acc, ext A), ext B) to Acc + (A - B), where ext is sext/zext.
static SDValue combinePExtWideningSubAcc(SDNode *N, SelectionDAG &DAG,
                                         const RISCVSubtarget &Subtarget) {
  using namespace SDPatternMatch;
 
  if (!Subtarget.hasStdExtP() || !Subtarget.is64Bit())
    return SDValue();
 
  if (N->getOpcode() != ISD::SUB)
    return SDValue();
 
  MVT VT = N->getSimpleValueType(0);
  if (VT != MVT::v4i16 && VT != MVT::v2i32)
    return SDValue();
 
  MVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
  auto Extend = [&](SDValue &Ext, SDValue &Src) {
    return m_Value(
        Ext, m_OneUse(m_AnyOf(m_SExt(m_Value(Src, m_SpecificVT(SrcVT))),
                              m_ZExt(m_Value(Src, m_SpecificVT(SrcVT))))));
  };
 
  SDValue Acc, ExtA, A, ExtB, B;
  if (!sd_match(N, m_Sub(m_OneUse(m_Add(m_Value(Acc), Extend(ExtA, A))),
                         Extend(ExtB, B))) ||
      ExtA.getOpcode() != ExtB.getOpcode())
    return SDValue();
 
  SDLoc DL(N);
  if (ExtA.getOpcode() == ISD::ZERO_EXTEND) {
    // No unsigned PM2 accumulate-subtract; keep the binary widening subtract
    // off the accumulator's dependency chain.
    return DAG.getNode(ISD::ADD, DL, VT, Acc,
                       DAG.getNode(ISD::SUB, DL, VT, ExtA, ExtB));
  }
 
  MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
  A = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, A, DAG.getUNDEF(SrcVT));
  B = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, B, DAG.getUNDEF(SrcVT));
 
  SDValue Zip = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, A, B);
  if (VT == MVT::v4i16) {
    SDValue ZipAsVT = DAG.getBitcast(VT, Zip);
    SDValue Low = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, ZipAsVT,
                              DAG.getValueType(MVT::v4i8));
    SDValue High = DAG.getNode(RISCVISD::PSRA, DL, VT, ZipAsVT,
                               DAG.getConstant(8, DL, MVT::i64));
    return DAG.getNode(ISD::ADD, DL, VT, Acc,
                       DAG.getNode(ISD::SUB, DL, VT, Low, High));
  }
 
  SDValue Ones = DAG.getConstant(1, DL, LegalSrcVT);
  return DAG.getNode(RISCVISD::PM2SUBA_H, DL, VT, Acc, Zip, Ones);
}
 
static SDValue performADDCombine(SDNode *N,
                                 TargetLowering::DAGCombinerInfo &DCI,
                                 const RISCVSubtarget &Subtarget) {
  SelectionDAG &DAG = DCI.DAG;
  if (SDValue V = combineAddOfBooleanXor(N, DAG))
    return V;
  if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
    return V;
  if (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) {
    if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
      return V;
    if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
      return V;
  }
  if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
    return V;
  if (SDValue V = combinePExtWideningAddAcc(N, DAG, Subtarget))
    return V;
  if (SDValue V = combinePExtWideningAddSub(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpOfZExt(N, DAG))
    return V;
  if (SDValue V = combineAddMulParts(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineAddMulh(N, DAG, Subtarget))
    return V;
 
  // fold (add (select lhs, rhs, cc, 0, y), x) ->
  //      (select lhs, rhs, cc, x, (add x, y))
  return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
}
 
// Try to turn a sub boolean RHS and constant LHS into an addi.
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // Require a constant LHS.
  auto *N0C = dyn_cast<ConstantSDNode>(N0);
  if (!N0C)
    return SDValue();
 
  // All our optimizations involve subtracting 1 from the immediate and forming
  // an ADDI. Make sure the new immediate is valid for an ADDI.
  APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
  if (!ImmValMinus1.isSignedIntN(12))
    return SDValue();
 
  SDValue NewLHS;
  if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
    // (sub constant, (setcc x, y, eq/neq)) ->
    // (add (setcc x, y, neq/eq), constant - 1)
    ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
    EVT SetCCOpVT = N1.getOperand(0).getValueType();
    if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.isInteger())
      return SDValue();
    CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT);
    NewLHS =
        DAG.getSetCC(SDLoc(N1), VT, N1.getOperand(0), N1.getOperand(1), CCVal);
  } else if (N1.getOpcode() == ISD::XOR && isOneConstant(N1.getOperand(1)) &&
             N1.getOperand(0).getOpcode() == ISD::SETCC) {
    // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
    // Since setcc returns a bool the xor is equivalent to 1-setcc.
    NewLHS = N1.getOperand(0);
  } else
    return SDValue();
 
  SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT);
  return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
}
 
// Looks for (sub (shl X, 8-Y), (shr X, Y)) where the Y-th bit in each byte is
// potentially set. It is fine for Y to be 0, meaning that (sub (shl X, 8), X)
// is also valid. Replace with (orc.b X). For example, 0b0000_1000_0000_1000 is
// valid with Y=3, while 0b0000_1000_0000_0100 is not.
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
                                     const RISCVSubtarget &Subtarget) {
  if (!Subtarget.hasStdExtZbb())
    return SDValue();
 
  EVT VT = N->getValueType(0);
 
  if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
 
  if (N0->getOpcode() != ISD::SHL)
    return SDValue();
 
  auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(N0.getOperand(1));
  if (!ShAmtCLeft)
    return SDValue();
  unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
 
  if (ShiftedAmount >= 8)
    return SDValue();
 
  SDValue LeftShiftOperand = N0->getOperand(0);
  SDValue RightShiftOperand = N1;
 
  if (ShiftedAmount != 0) { // Right operand must be a right shift.
    if (N1->getOpcode() != ISD::SRL)
      return SDValue();
    auto *ShAmtCRight = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
      return SDValue();
    RightShiftOperand = N1.getOperand(0);
  }
 
  // At least one shift should have a single use.
  if (!N0.hasOneUse() && (ShiftedAmount == 0 || !N1.hasOneUse()))
    return SDValue();
 
  if (LeftShiftOperand != RightShiftOperand)
    return SDValue();
 
  APInt Mask = APInt::getSplat(VT.getSizeInBits(), APInt(8, 0x1));
  Mask <<= ShiftedAmount;
  // Check that X has indeed the right shape (only the Y-th bit can be set in
  // every byte).
  if (!DAG.MaskedValueIsZero(LeftShiftOperand, ~Mask))
    return SDValue();
 
  return DAG.getNode(RISCVISD::ORC_B, SDLoc(N), VT, LeftShiftOperand);
}
 
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  if (SDValue V = combineSubOfBoolean(N, DAG))
    return V;
 
  EVT VT = N->getValueType(0);
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
  if (isNullConstant(N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
      isNullConstant(N1.getOperand(1)) &&
      N1.getValueType() == N1.getOperand(0).getValueType()) {
    ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
    if (CCVal == ISD::SETLT) {
      SDLoc DL(N);
      unsigned ShAmt = N0.getValueSizeInBits() - 1;
      return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
                         DAG.getConstant(ShAmt, DL, VT));
    }
  }
 
  // fold (sub 0, (srl (and X, (1 << ShAmt)), ShAmt)) ->
  //      (sra (shl X, ShAmt2), bits - 1)
  // where ShAmt2 = (bits - 1) - ShAmt. This extracts bit ShAmt of X and
  // sign-extends it across the whole register.
  {
    using namespace SDPatternMatch;
    SDValue X;
    uint64_t Mask, ShAmt;
    if (isNullConstant(N0) &&
        sd_match(N1,
                 m_OneUse(m_Srl(m_OneUse(m_And(m_Value(X), m_ConstInt(Mask))),
                                m_ConstInt(ShAmt)))) &&
        isPowerOf2_64(Mask) && Log2_64(Mask) == ShAmt) {
      SDLoc DL(N);
      unsigned Bits = VT.getSizeInBits();
      unsigned ShAmt2 = Bits - 1 - ShAmt;
      SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
                                DAG.getShiftAmountConstant(ShAmt2, VT, DL));
      return DAG.getNode(ISD::SRA, DL, VT, Shl,
                         DAG.getShiftAmountConstant(Bits - 1, VT, DL));
    }
  }
 
  if (SDValue V = combinePExtWideningSubAcc(N, DAG, Subtarget))
    return V;
  if (SDValue V = combinePExtWideningAddSub(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpOfZExt(N, DAG))
    return V;
  if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget))
    return V;
 
  // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
  //      (select lhs, rhs, cc, x, (sub x, y))
  return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false, Subtarget);
}
 
// Apply DeMorgan's law to (and/or (xor X, 1), (xor Y, 1)) if X and Y are 0/1.
// Legalizing setcc can introduce xors like this. Doing this transform reduces
// the number of xors and may allow the xor to fold into a branch condition.
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  bool IsAnd = N->getOpcode() == ISD::AND;
 
  if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR)
    return SDValue();
 
  if (!N0.hasOneUse() || !N1.hasOneUse())
    return SDValue();
 
  SDValue N01 = N0.getOperand(1);
  SDValue N11 = N1.getOperand(1);
 
  // For AND, SimplifyDemandedBits may have turned one of the (xor X, 1) into
  // (xor X, -1) based on the upper bits of the other operand being 0. If the
  // operation is And, allow one of the Xors to use -1.
  if (isOneConstant(N01)) {
    if (!isOneConstant(N11) && !(IsAnd && isAllOnesConstant(N11)))
      return SDValue();
  } else if (isOneConstant(N11)) {
    // N01 and N11 being 1 was already handled. Handle N11==1 and N01==-1.
    if (!(IsAnd && isAllOnesConstant(N01)))
      return SDValue();
  } else
    return SDValue();
 
  EVT VT = N->getValueType(0);
 
  SDValue N00 = N0.getOperand(0);
  SDValue N10 = N1.getOperand(0);
 
  // The LHS of the xors needs to be 0/1.
  APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
  if (!DAG.MaskedValueIsZero(N00, Mask) || !DAG.MaskedValueIsZero(N10, Mask))
    return SDValue();
 
  // Invert the opcode and insert a new xor.
  SDLoc DL(N);
  unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
  SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10);
  return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
}
 
// Handle P extension truncate patterns, both on packed vectors and on scalar
// i32 (the RV32-only asub/asubu and mulhr* instructions):
// ASUB/ASUBU: (trunc (srl (sub ([s|z]ext a), ([s|z]ext b)), 1))
// MULHSU: (trunc (srl (mul (sext a), (zext b)), EltBits))
// MULHR*: (trunc (srl (add (mul (sext a), (zext b)), round_const), EltBits))
static SDValue combinePExtTruncate(SDNode *N, SelectionDAG &DAG,
                                   const RISCVSubtarget &Subtarget) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  if (N0.getOpcode() != ISD::SRL)
    return SDValue();
 
  if (VT != MVT::v4i16 && VT != MVT::v2i16 && VT != MVT::v8i8 &&
      VT != MVT::v4i8 && VT != MVT::v2i32 && VT != MVT::i32)
    return SDValue();
 
  ConstantSDNode *C = isConstOrConstSplat(N0.getOperand(1));
  if (!C)
    return SDValue();
 
  SDValue Op = N0.getOperand(0);
  unsigned ShAmtVal = C->getZExtValue();
  unsigned EltBits = VT.getScalarSizeInBits();
 
  // Check for rounding pattern: (add (mul ...), round_const)
  bool IsRounding = false;
  if (Op.getOpcode() == ISD::ADD && (EltBits == 16 || EltBits == 32)) {
    ConstantSDNode *RndC = isConstOrConstSplat(Op.getOperand(1));
    if (RndC && RndC->getZExtValue() == (1ULL << (EltBits - 1)) &&
        Op.getOperand(0).getOpcode() == ISD::MUL) {
      Op = Op.getOperand(0);
      IsRounding = true;
    }
  }
 
  // Ensure Op is a binary operation before accessing its operands.
  if (Op.getNumOperands() != 2)
    return SDValue();
 
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
 
  bool LHSIsSExt = LHS.getOpcode() == ISD::SIGN_EXTEND;
  bool LHSIsZExt = LHS.getOpcode() == ISD::ZERO_EXTEND;
  bool RHSIsSExt = RHS.getOpcode() == ISD::SIGN_EXTEND;
  bool RHSIsZExt = RHS.getOpcode() == ISD::ZERO_EXTEND;
 
  if (!(LHSIsSExt || LHSIsZExt) || !(RHSIsSExt || RHSIsZExt))
    return SDValue();
 
  SDValue A = LHS.getOperand(0);
  SDValue B = RHS.getOperand(0);
 
  if (A.getValueType() != VT || B.getValueType() != VT)
    return SDValue();
 
  unsigned Opc;
  switch (Op.getOpcode()) {
  default:
    return SDValue();
  case ISD::SUB:
    // PASUB/PASUBU: shift amount must be 1
    if (ShAmtVal != 1)
      return SDValue();
    if (LHSIsSExt && RHSIsSExt)
      Opc = RISCVISD::ASUB;
    else if (LHSIsZExt && RHSIsZExt)
      Opc = RISCVISD::ASUBU;
    else
      return SDValue();
    break;
  case ISD::MUL:
    // MULH*/MULHR*: shift amount must be element size, only for i16/i32
    if (ShAmtVal != EltBits || (EltBits != 16 && EltBits != 32))
      return SDValue();
    if (!Subtarget.is64Bit() && VT == MVT::v4i16)
      return SDValue();
    if (IsRounding) {
      if (LHSIsSExt && RHSIsSExt) {
        Opc = RISCVISD::MULHR;
      } else if (LHSIsZExt && RHSIsZExt) {
        Opc = RISCVISD::MULHRU;
      } else if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
        Opc = RISCVISD::MULHRSU;
        // commuted case
        if (LHSIsZExt && RHSIsSExt)
          std::swap(A, B);
      } else {
        return SDValue();
      }
    } else {
      // Scalar mulhsu is handled elsewhere, only match the packed MULHSU here.
      if (!VT.isVector())
        return SDValue();
      if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
        Opc = RISCVISD::MULHSU;
        // commuted case
        if (LHSIsZExt && RHSIsSExt)
          std::swap(A, B);
      } else
        return SDValue();
    }
 
    // RV32 has scalar rounded multiply-high instructions, but no paired form.
    // Split v2i32 while the widening multiply shape is still visible.
    if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
      if (!IsRounding)
        return SDValue();
      SDLoc DL(N);
      SDValue ALo = DAG.getExtractVectorElt(DL, MVT::i32, A, 0);
      SDValue AHi = DAG.getExtractVectorElt(DL, MVT::i32, A, 1);
      SDValue BLo = DAG.getExtractVectorElt(DL, MVT::i32, B, 0);
      SDValue BHi = DAG.getExtractVectorElt(DL, MVT::i32, B, 1);
      SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, ALo, BLo);
      SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, AHi, BHi);
      return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
    }
 
    // On RV64, v8i8 MULHSU is built from a pair of widening byte multiplies
    // (picking out the even/odd result lanes) recombined with PPAIRO.
    if (Subtarget.is64Bit() && VT == MVT::v8i8 && Opc == RISCVISD::MULHSU) {
      SDLoc DL(N);
      SDValue Lo =
          DAG.getNode(RISCVISD::PMULSU_HALVES_00, DL, MVT::v4i16, A, B);
      SDValue Hi =
          DAG.getNode(RISCVISD::PMULSU_HALVES_11, DL, MVT::v4i16, A, B);
      return DAG.getNode(RISCVISD::PPAIRO, DL, VT, DAG.getBitcast(VT, Lo),
                         DAG.getBitcast(VT, Hi));
    }
    break;
  }
 
  return DAG.getNode(Opc, SDLoc(N), VT, {A, B});
}
 
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
 
  // P truncate patterns: packed vectors, plus RV32-only scalar i32.
  if (Subtarget.hasStdExtP() &&
      (VT.isFixedLengthVector() || (VT == MVT::i32 && !Subtarget.is64Bit())))
    return combinePExtTruncate(N, DAG, Subtarget);
 
  // Pre-promote (i1 (truncate (srl X, Y))) on RV64 with Zbs without zero
  // extending X. This is safe since we only need the LSB after the shift and
  // shift amounts larger than 31 would produce poison. If we wait until
  // type legalization, we'll create RISCVISD::SRLW and we can't recover it
  // to use a BEXT instruction.
  if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
      N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL &&
      !isa<ConstantSDNode>(N0.getOperand(1)) && N0.hasOneUse()) {
    SDLoc DL(N0);
    SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
    SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
    SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
    return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl);
  }
 
  return SDValue();
}
 
// InstCombinerImpl::transformZExtICmp will narrow a zext of an icmp with a
// truncation. But RVV doesn't have truncation instructions for more than twice
// the bitwidth.
//
// E.g. trunc <vscale x 1 x i64> %x to <vscale x 1 x i8> will generate:
//
//     vsetvli a0, zero, e32, m2, ta, ma
//     vnsrl.wi v12, v8, 0
//     vsetvli zero, zero, e16, m1, ta, ma
//     vnsrl.wi v8, v12, 0
//     vsetvli zero, zero, e8, mf2, ta, ma
//     vnsrl.wi v8, v8, 0
//
// So reverse the combine so we generate an vmseq/vmsne again:
//
// and (lshr (trunc X), ShAmt), 1
// -->
// zext (icmp ne (and X, (1 << ShAmt)), 0)
//
// and (lshr (not (trunc X)), ShAmt), 1
// -->
// zext (icmp eq (and X, (1 << ShAmt)), 0)
static SDValue reverseZExtICmpCombine(SDNode *N, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  using namespace SDPatternMatch;
  SDLoc DL(N);
 
  if (!Subtarget.hasVInstructions())
    return SDValue();
 
  EVT VT = N->getValueType(0);
  if (!VT.isVector())
    return SDValue();
 
  APInt ShAmt;
  SDValue Inner;
  if (!sd_match(N, m_And(m_OneUse(m_Srl(m_Value(Inner), m_ConstInt(ShAmt))),
                         m_One())))
    return SDValue();
 
  SDValue X;
  bool IsNot;
  if (sd_match(Inner, m_Not(m_Trunc(m_Value(X)))))
    IsNot = true;
  else if (sd_match(Inner, m_Trunc(m_Value(X))))
    IsNot = false;
  else
    return SDValue();
 
  EVT WideVT = X.getValueType();
  if (VT.getScalarSizeInBits() >= WideVT.getScalarSizeInBits() / 2)
    return SDValue();
 
  SDValue Res =
      DAG.getNode(ISD::AND, DL, WideVT, X,
                  DAG.getConstant(1ULL << ShAmt.getZExtValue(), DL, WideVT));
  Res = DAG.getSetCC(DL,
                     EVT::getVectorVT(*DAG.getContext(), MVT::i1,
                                      WideVT.getVectorElementCount()),
                     Res, DAG.getConstant(0, DL, WideVT),
                     IsNot ? ISD::SETEQ : ISD::SETNE);
  return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res);
}
 
static SDValue reduceANDOfAtomicLoad(SDNode *N,
                                     TargetLowering::DAGCombinerInfo &DCI) {
  SelectionDAG &DAG = DCI.DAG;
  if (N->getOpcode() != ISD::AND)
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  if (N0.getOpcode() != ISD::ATOMIC_LOAD)
    return SDValue();
  if (!N0.hasOneUse())
    return SDValue();
 
  AtomicSDNode *ALoad = cast<AtomicSDNode>(N0.getNode());
  if (isStrongerThanMonotonic(ALoad->getSuccessOrdering()))
    return SDValue();
 
  EVT LoadedVT = ALoad->getMemoryVT();
  ConstantSDNode *MaskConst = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!MaskConst)
    return SDValue();
  uint64_t Mask = MaskConst->getZExtValue();
  uint64_t ExpectedMask = maskTrailingOnes<uint64_t>(LoadedVT.getSizeInBits());
  if (Mask != ExpectedMask)
    return SDValue();
 
  SDValue ZextLoad = DAG.getAtomicLoad(
      ISD::ZEXTLOAD, SDLoc(N), ALoad->getMemoryVT(), N->getValueType(0),
      ALoad->getChain(), ALoad->getBasePtr(), ALoad->getMemOperand());
  DCI.CombineTo(N, ZextLoad);
  DAG.ReplaceAllUsesOfValueWith(SDValue(N0.getNode(), 1), ZextLoad.getValue(1));
  DCI.recursivelyDeleteUnusedNodes(N0.getNode());
  return SDValue(N, 0);
}
 
// Sometimes a mask is applied after a shift. If that shift was fed by a
// load, there is sometimes the opportunity to narrow the load, which is
// hidden by the intermediate shift. Detect that case and commute the
// shift/and in order to enable load narrowing.
static SDValue combineNarrowableShiftedLoad(SDNode *N, SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
  if (!VT.isScalarInteger())
    return SDValue();
 
  using namespace SDPatternMatch;
  SDValue LoadNode;
  APInt MaskVal, ShiftVal;
  // (and (shl (load ...), ShiftAmt), Mask)
  if (!sd_match(
          N, m_And(m_OneUse(m_Shl(m_Value(LoadNode, m_SpecificOpc<ISD::LOAD>()),
                                  m_ConstInt(ShiftVal))),
                   m_ConstInt(MaskVal)))) {
    return SDValue();
  }
 
  uint64_t ShiftAmt = ShiftVal.getZExtValue();
 
  if (ShiftAmt >= VT.getSizeInBits())
    return SDValue();
 
  // Calculate the appropriate mask if it were applied before the shift.
  APInt InnerMask = MaskVal.lshr(ShiftAmt);
  bool IsNarrowable =
      InnerMask == 0xff || InnerMask == 0xffff || InnerMask == 0xffffffff;
 
  if (!IsNarrowable)
    return SDValue();
 
  // AND the loaded value and change the shift appropriately, allowing
  // the load to be narrowed.
  SDLoc DL(N);
  SDValue InnerAnd = DAG.getNode(ISD::AND, DL, VT, LoadNode,
                                 DAG.getConstant(InnerMask, DL, VT));
  return DAG.getNode(ISD::SHL, DL, VT, InnerAnd,
                     DAG.getShiftAmountConstant(ShiftAmt, VT, DL));
}
 
// Combines two comparison operation and logic operation to one selection
// operation(min, max) and logic operation. Returns new constructed Node if
// conditions for optimization are satisfied.
static SDValue performANDCombine(SDNode *N,
                                 TargetLowering::DAGCombinerInfo &DCI,
                                 const RISCVSubtarget &Subtarget) {
  SelectionDAG &DAG = DCI.DAG;
  SDValue N0 = N->getOperand(0);
 
  // Pre-promote (i32 (and (srl X, Y), 1)) on RV64 with Zbs without zero
  // extending X. This is safe since we only need the LSB after the shift and
  // shift amounts larger than 31 would produce poison. If we wait until
  // type legalization, we'll create RISCVISD::SRLW and we can't recover it
  // to use a BEXT instruction.
  if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
      N->getValueType(0) == MVT::i32 && isOneConstant(N->getOperand(1)) &&
      N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(N0.getOperand(1)) &&
      N0.hasOneUse()) {
    SDLoc DL(N);
    SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
    SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
    SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
    SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl,
                              DAG.getConstant(1, DL, MVT::i64));
    return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
  }
 
  if (SDValue V = combineNarrowableShiftedLoad(N, DAG))
    return V;
  if (SDValue V = reverseZExtICmpCombine(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
    return V;
  if (SDValue V = reduceANDOfAtomicLoad(N, DCI))
    return V;
 
  if (DCI.isAfterLegalizeDAG())
    if (SDValue V = combineDeMorganOfBoolean(N, DAG))
      return V;
 
  // fold (and (select lhs, rhs, cc, -1, y), x) ->
  //      (select lhs, rhs, cc, x, (and x, y))
  return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget);
}
 
// Try to pull an xor with 1 through a select idiom that uses czero_eqz/nez.
// FIXME: Generalize to other binary operators with same operand.
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1,
                                SelectionDAG &DAG) {
  assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
 
  if (N0.getOpcode() != RISCVISD::CZERO_EQZ ||
      N1.getOpcode() != RISCVISD::CZERO_NEZ ||
      !N0.hasOneUse() || !N1.hasOneUse())
    return SDValue();
 
  // Should have the same condition.
  SDValue Cond = N0.getOperand(1);
  if (Cond != N1.getOperand(1))
    return SDValue();
 
  SDValue TrueV = N0.getOperand(0);
  SDValue FalseV = N1.getOperand(0);
 
  if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR ||
      TrueV.getOperand(1) != FalseV.getOperand(1) ||
      !isOneConstant(TrueV.getOperand(1)) ||
      !TrueV.hasOneUse() || !FalseV.hasOneUse())
    return SDValue();
 
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0),
                              Cond);
  SDValue NewN1 =
      DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0), Cond);
  SDValue NewOr =
      DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1, SDNodeFlags::Disjoint);
  return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1));
}
 
// (xor X, (xor (and X, C2), Y))
// ->(qc_insb X, (sra Y, ShAmt), Width, ShAmt)
// where C2 is a shifted mask with width = Width and shift = ShAmt
// qc_insb might become qc.insb or qc.insbi depending on the operands.
static SDValue combineXorToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
                                          const RISCVSubtarget &Subtarget) {
  if (!Subtarget.hasVendorXqcibm())
    return SDValue();
 
  using namespace SDPatternMatch;
  SDValue Base, Inserted;
  APInt CMask;
  if (!sd_match(N, m_Xor(m_Value(Base),
                         m_OneUse(m_Xor(m_OneUse(m_And(m_Deferred(Base),
                                                       m_ConstInt(CMask))),
                                        m_Value(Inserted))))))
    return SDValue();
 
  if (N->getValueType(0) != MVT::i32)
    return SDValue();
  unsigned Width, ShAmt;
  if (!CMask.isShiftedMask(ShAmt, Width))
    return SDValue();
 
  // Check if all zero bits in CMask are also zero in Inserted
  if (!DAG.MaskedValueIsZero(Inserted, ~CMask))
    return SDValue();
 
  SDLoc DL(N);
 
  // `Inserted` needs to be right shifted before it is put into the
  // instruction.
  Inserted = DAG.getNode(ISD::SRA, DL, MVT::i32, Inserted,
                         DAG.getShiftAmountConstant(ShAmt, MVT::i32, DL));
 
  SDValue Ops[] = {Base, Inserted, DAG.getConstant(Width, DL, MVT::i32),
                   DAG.getConstant(ShAmt, DL, MVT::i32)};
  return DAG.getNode(RISCVISD::QC_INSB, DL, MVT::i32, Ops);
}
 
static SDValue combineOrToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
                                         const RISCVSubtarget &Subtarget) {
  if (!Subtarget.hasVendorXqcibm())
    return SDValue();
 
  using namespace SDPatternMatch;
 
  SDValue X;
  APInt MaskImm;
  if (!sd_match(N, m_Or(m_OneUse(m_Value(X)), m_ConstInt(MaskImm))))
    return SDValue();
 
  unsigned ShAmt, Width;
  if (!MaskImm.isShiftedMask(ShAmt, Width) || MaskImm.isSignedIntN(12))
    return SDValue();
 
  if (N->getValueType(0) != MVT::i32)
    return SDValue();
 
  // If Zbs is enabled and it is a single bit set we can use BSETI which
  // can be compressed to C_BSETI when Xqcibm in enabled.
  if (Width == 1 && Subtarget.hasStdExtZbs())
    return SDValue();
 
  // If C1 is a shifted mask (but can't be formed as an ORI),
  // use a bitfield insert of -1.
  // Transform (or x, C1)
  //        -> (qc.insbi x, -1, width, shift)
  SDLoc DL(N);
 
  SDValue Ops[] = {X, DAG.getSignedConstant(-1, DL, MVT::i32),
                   DAG.getConstant(Width, DL, MVT::i32),
                   DAG.getConstant(ShAmt, DL, MVT::i32)};
  return DAG.getNode(RISCVISD::QC_INSB, DL, MVT::i32, Ops);
}
 
// Generate a QC_INSB/QC_INSBI from 'or (and X, MaskImm), OrImm' iff the value
// being inserted only sets known zero bits.
static SDValue combineOrAndToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
                                            const RISCVSubtarget &Subtarget) {
  // Supported only in Xqcibm for now.
  if (!Subtarget.hasVendorXqcibm())
    return SDValue();
 
  using namespace SDPatternMatch;
 
  SDValue Inserted;
  APInt MaskImm, OrImm;
  if (!sd_match(
          N, m_SpecificVT(MVT::i32, m_Or(m_OneUse(m_And(m_Value(Inserted),
                                                        m_ConstInt(MaskImm))),
                                         m_ConstInt(OrImm)))))
    return SDValue();
 
  // Compute the Known Zero for the AND as this allows us to catch more general
  // cases than just looking for AND with imm.
  KnownBits Known = DAG.computeKnownBits(N->getOperand(0));
 
  // The bits being inserted must only set those bits that are known to be
  // zero.
  if (!OrImm.isSubsetOf(Known.Zero)) {
    // FIXME:  It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
    // currently handle this case.
    return SDValue();
  }
 
  unsigned ShAmt, Width;
  // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
  if (!Known.Zero.isShiftedMask(ShAmt, Width))
    return SDValue();
 
  // QC_INSB(I) dst, src, #width, #shamt.
  SDLoc DL(N);
 
  SDValue ImmNode =
      DAG.getSignedConstant(OrImm.getSExtValue() >> ShAmt, DL, MVT::i32);
 
  SDValue Ops[] = {Inserted, ImmNode, DAG.getConstant(Width, DL, MVT::i32),
                   DAG.getConstant(ShAmt, DL, MVT::i32)};
  return DAG.getNode(RISCVISD::QC_INSB, DL, MVT::i32, Ops);
}
 
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                const RISCVSubtarget &Subtarget) {
  SelectionDAG &DAG = DCI.DAG;
 
  if (SDValue V = combineOrAndToBitfieldInsert(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
    return V;
 
  if (DCI.isAfterLegalizeDAG()) {
    if (SDValue V = combineOrToBitfieldInsert(N, DAG, Subtarget))
      return V;
    if (SDValue V = combineDeMorganOfBoolean(N, DAG))
      return V;
  }
 
  // Look for Or of CZERO_EQZ/NEZ with same condition which is the select idiom.
  // We may be able to pull a common operation out of the true and false value.
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (SDValue V = combineOrOfCZERO(N, N0, N1, DAG))
    return V;
  if (SDValue V = combineOrOfCZERO(N, N1, N0, DAG))
    return V;
 
  // fold (or (select cond, 0, y), x) ->
  //      (select cond, x, (or x, y))
  return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
}
 
// Fold (vmxor_vl (setcc_vl a, b, cc), vmset_vl) -> (setcc_vl a, b, !cc) for
// integer compares. The NOT also flips the masked-off lanes, which the inverted
// compare copies unchanged from the passthru. So the passthru must be undef,
// but any mask is fine.
static SDValue combineVMNOTOfSetCC(SDNode *N, SelectionDAG &DAG) {
  SDValue Cmp = N->getOperand(0);
  if (Cmp.getOpcode() != RISCVISD::VMSET_VL &&
      N->getOperand(1).getOpcode() != RISCVISD::VMSET_VL)
    return SDValue();
 
  if (Cmp.getOpcode() == RISCVISD::VMSET_VL)
    Cmp = N->getOperand(1);
 
  if (Cmp.getOpcode() != RISCVISD::SETCC_VL || !Cmp.hasOneUse() ||
      !Cmp.getOperand(0).getValueType().isInteger() ||
      !Cmp.getOperand(3).isUndef())
    return SDValue();
 
  ISD::CondCode CC = cast<CondCodeSDNode>(Cmp.getOperand(2))->get();
  CC = ISD::getSetCCInverse(CC, Cmp.getOperand(0).getValueType());
  return DAG.getNode(RISCVISD::SETCC_VL, SDLoc(N), Cmp.getValueType(),
                     {Cmp.getOperand(0), Cmp.getOperand(1), DAG.getCondCode(CC),
                      Cmp.getOperand(3), Cmp.getOperand(4), Cmp.getOperand(5)});
}
 
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
 
  // Pre-promote (i32 (xor (shl -1, X), ~0)) on RV64 with Zbs so we can use
  // (ADDI (BSET X0, X), -1). If we wait until type legalization, we'll create
  // RISCVISD:::SLLW and we can't recover it to use a BSET instruction.
  if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
      N->getValueType(0) == MVT::i32 && isAllOnesConstant(N1) &&
      N0.getOpcode() == ISD::SHL && isAllOnesConstant(N0.getOperand(0)) &&
      !isa<ConstantSDNode>(N0.getOperand(1)) && N0.hasOneUse()) {
    SDLoc DL(N);
    SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
    SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
    SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1);
    SDValue Not = DAG.getNOT(DL, Shl, MVT::i64);
    return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Not);
  }
 
  // fold (xor (sllw 1, x), -1) -> (rolw ~1, x)
  // NOTE: Assumes ROL being legal means ROLW is legal.
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (N0.getOpcode() == RISCVISD::SLLW &&
      isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0)) &&
      TLI.isOperationLegal(ISD::ROTL, MVT::i64)) {
    SDLoc DL(N);
    return DAG.getNode(RISCVISD::ROLW, DL, MVT::i64,
                       DAG.getConstant(~1, DL, MVT::i64), N0.getOperand(1));
  }
 
  // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
  if (N0.getOpcode() == ISD::SETCC && isOneConstant(N1) && N0.hasOneUse()) {
    auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.getOperand(0));
    ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    if (ConstN00 && CC == ISD::SETLT) {
      EVT VT = N0.getValueType();
      SDLoc DL(N0);
      const APInt &Imm = ConstN00->getAPIntValue();
      if ((Imm + 1).isSignedIntN(12))
        return DAG.getSetCC(DL, VT, N0.getOperand(1),
                            DAG.getConstant(Imm + 1, DL, VT), CC);
    }
  }
 
  if (SDValue V = combineXorToBitfieldInsert(N, DAG, Subtarget))
    return V;
 
  if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
    return V;
  if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
    return V;
 
  // fold (xor (select cond, 0, y), x) ->
  //      (select cond, x, (xor x, y))
  return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
}
 
// Try to expand a multiply to a sequence of shifts and add/subs,
// for a machine without native mul instruction.
static SDValue expandMulToNAFSequence(SDNode *N, SelectionDAG &DAG,
                                      uint64_t MulAmt) {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  const uint64_t BitWidth = VT.getFixedSizeInBits();
 
  SDValue Result = DAG.getConstant(0, DL, N->getValueType(0));
  SDValue N0 = N->getOperand(0);
 
  // Find the Non-adjacent form of the multiplier.
  for (uint64_t E = MulAmt, I = 0; E && I < BitWidth; ++I, E >>= 1) {
    if (E & 1) {
      bool IsAdd = (E & 3) == 1;
      E -= IsAdd ? 1 : -1;
      SDValue ShiftVal = DAG.getNode(ISD::SHL, DL, VT, N0,
                                     DAG.getShiftAmountConstant(I, VT, DL));
      ISD::NodeType AddSubOp = IsAdd ? ISD::ADD : ISD::SUB;
      Result = DAG.getNode(AddSubOp, DL, VT, Result, ShiftVal);
    }
  }
 
  return Result;
}
 
// X * (2^N +/- 2^M) -> (add/sub (shl X, C1), (shl X, C2))
static SDValue expandMulToAddOrSubOfShl(SDNode *N, SelectionDAG &DAG,
                                        uint64_t MulAmt) {
  uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
  SDValue X = N->getOperand(0);
  ISD::NodeType Op;
  uint64_t ShiftAmt1;
  bool CanSub = isPowerOf2_64(MulAmt + MulAmtLowBit);
  auto PreferSub = [X, MulAmtLowBit]() {
    // For MulAmt == 3 << M both (X << M + 2) - (X << M)
    // and (X << M + 1) + (X << M) are valid expansions.
    // Prefer SUB if we can get (X << M + 2) for free,
    // because X is exact (Y >> M + 2).
    uint64_t ShAmt = Log2_64(MulAmtLowBit) + 2;
    using namespace SDPatternMatch;
    return sd_match(X, m_ExactSr(m_Value(), m_SpecificInt(ShAmt)));
  };
  if (isPowerOf2_64(MulAmt - MulAmtLowBit) && !(CanSub && PreferSub())) {
    Op = ISD::ADD;
    ShiftAmt1 = MulAmt - MulAmtLowBit;
  } else if (CanSub) {
    Op = ISD::SUB;
    ShiftAmt1 = MulAmt + MulAmtLowBit;
  } else {
    return SDValue();
  }
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, X,
                               DAG.getConstant(Log2_64(ShiftAmt1), DL, VT));
  SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, X,
                               DAG.getConstant(Log2_64(MulAmtLowBit), DL, VT));
  return DAG.getNode(Op, DL, VT, Shift1, Shift2);
}
 
static SDValue getShlAddShlAdd(SDNode *N, SelectionDAG &DAG, unsigned ShX,
                               unsigned ShY, bool AddX, unsigned Shift) {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  SDValue X = N->getOperand(0);
  // Put the shift first if we can fold:
  // a. a zext into the shift forming a slli.uw
  // b. an exact shift right forming one shorter shift or no shift at all
  using namespace SDPatternMatch;
  if (Shift != 0 &&
      sd_match(X, m_AnyOf(m_And(m_Value(), m_SpecificInt(UINT64_C(0xffffffff))),
                          m_ExactSr(m_Value(), m_ConstInt())))) {
    X = DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(Shift, DL, VT));
    Shift = 0;
  }
  SDValue ShlAdd = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
                               DAG.getTargetConstant(ShY, DL, VT), X);
  if (ShX != 0)
    ShlAdd = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, ShlAdd,
                         DAG.getTargetConstant(ShX, DL, VT), AddX ? X : ShlAdd);
  if (Shift == 0)
    return ShlAdd;
  // Otherwise, put the shl last so that it can fold with following instructions
  // (e.g. sext or add).
  return DAG.getNode(ISD::SHL, DL, VT, ShlAdd, DAG.getConstant(Shift, DL, VT));
}
 
static SDValue expandMulToShlAddShlAdd(SDNode *N, SelectionDAG &DAG,
                                       uint64_t MulAmt, unsigned Shift) {
  switch (MulAmt) {
  // 3/5/9 -> (shYadd X, X)
  case 3:
    return getShlAddShlAdd(N, DAG, 0, 1, /*AddX=*/false, Shift);
  case 5:
    return getShlAddShlAdd(N, DAG, 0, 2, /*AddX=*/false, Shift);
  case 9:
    return getShlAddShlAdd(N, DAG, 0, 3, /*AddX=*/false, Shift);
  // 3/5/9 * 3/5/9 -> (shXadd (shYadd X, X), (shYadd X, X))
  case 5 * 3:
    return getShlAddShlAdd(N, DAG, 2, 1, /*AddX=*/false, Shift);
  case 9 * 3:
    return getShlAddShlAdd(N, DAG, 3, 1, /*AddX=*/false, Shift);
  case 5 * 5:
    return getShlAddShlAdd(N, DAG, 2, 2, /*AddX=*/false, Shift);
  case 9 * 5:
    return getShlAddShlAdd(N, DAG, 3, 2, /*AddX=*/false, Shift);
  case 9 * 9:
    return getShlAddShlAdd(N, DAG, 3, 3, /*AddX=*/false, Shift);
  default:
    break;
  }
 
  int ShX;
  if (int ShY = isShifted359(MulAmt - 1, ShX)) {
    assert(ShX != 0 && "MulAmt=4,6,10 handled before");
    // 2/4/8 * 3/5/9 + 1 -> (shXadd (shYadd X, X), X)
    if (ShX <= 3)
      return getShlAddShlAdd(N, DAG, ShX, ShY, /*AddX=*/true, Shift);
    // 2^N * 3/5/9 + 1 -> (add (shYadd (shl X, N), (shl X, N)), X)
    if (Shift == 0) {
      SDLoc DL(N);
      EVT VT = N->getValueType(0);
      SDValue X = N->getOperand(0);
      SDValue Shl =
          DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShX, DL, VT));
      SDValue ShlAdd = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Shl,
                                   DAG.getTargetConstant(ShY, DL, VT), Shl);
      return DAG.getNode(ISD::ADD, DL, VT, ShlAdd, X);
    }
  }
  return SDValue();
}
 
// Try to expand a scalar multiply to a faster sequence.
static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
                         TargetLowering::DAGCombinerInfo &DCI,
                         const RISCVSubtarget &Subtarget) {
 
  EVT VT = N->getValueType(0);
 
  // LI + MUL is usually smaller than the alternative sequence.
  if (DAG.getMachineFunction().getFunction().hasMinSize())
    return SDValue();
 
  if (VT != Subtarget.getXLenVT())
    return SDValue();
 
  bool ShouldExpandMul =
      (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) ||
      !Subtarget.hasStdExtZmmul();
  if (!ShouldExpandMul)
    return SDValue();
 
  ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!CNode)
    return SDValue();
  uint64_t MulAmt = CNode->getZExtValue();
 
  // Don't do this if the Xqciac extension is enabled and the MulAmt in simm12.
  if (Subtarget.hasVendorXqciac() && isInt<12>(CNode->getSExtValue()))
    return SDValue();
 
  // WARNING: The code below is knowingly incorrect with regards to undef
  // semantics.  We're adding additional uses of X here, and in principle, we
  // should be freezing X before doing so.  However, adding freeze here causes
  // real regressions, and no other target properly freezes X in these cases
  // either.
  if (Subtarget.hasShlAdd(3)) {
    // 3/5/9 * 2^N -> (shl (shXadd X, X), N)
    // 3/5/9 * 3/5/9 * 2^N - In particular, this covers multiples
    // of 25 which happen to be quite common.
    // (2/4/8 * 3/5/9 + 1) * 2^N
    unsigned Shift = llvm::countr_zero(MulAmt);
    if (SDValue V = expandMulToShlAddShlAdd(N, DAG, MulAmt >> Shift, Shift))
      return V;
 
    // If this is a power 2 + 2/4/8, we can use a shift followed by a single
    // shXadd. First check if this a sum of two power of 2s because that's
    // easy. Then count how many zeros are up to the first bit.
    SDValue X = N->getOperand(0);
    if (Shift >= 1 && Shift <= 3 && isPowerOf2_64(MulAmt & (MulAmt - 1))) {
      unsigned ShiftAmt = llvm::countr_zero((MulAmt & (MulAmt - 1)));
      SDLoc DL(N);
      SDValue Shift1 =
          DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
      return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
                         DAG.getTargetConstant(Shift, DL, VT), Shift1);
    }
 
    // TODO: 2^(C1>3) * 3/5/9 - 1
 
    // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
    if (MulAmt > 2 && isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
      unsigned ScaleShift = llvm::countr_zero(MulAmt - 1);
      if (ScaleShift >= 1 && ScaleShift < 4) {
        unsigned ShiftAmt = llvm::countr_zero((MulAmt - 1) & (MulAmt - 2));
        SDLoc DL(N);
        SDValue Shift1 =
            DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
        return DAG.getNode(
            ISD::ADD, DL, VT, Shift1,
            DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
                        DAG.getTargetConstant(ScaleShift, DL, VT), X));
      }
    }
 
    // 2^N - 3/5/9 --> (sub (shl X, C1), (shXadd X, x))
    for (uint64_t Offset : {3, 5, 9}) {
      if (isPowerOf2_64(MulAmt + Offset)) {
        unsigned ShAmt = llvm::countr_zero(MulAmt + Offset);
        if (ShAmt >= VT.getSizeInBits())
          continue;
        SDLoc DL(N);
        SDValue Shift1 =
            DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShAmt, DL, VT));
        SDValue Mul359 =
            DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
                        DAG.getTargetConstant(Log2_64(Offset - 1), DL, VT), X);
        return DAG.getNode(ISD::SUB, DL, VT, Shift1, Mul359);
      }
    }
  }
 
  if (SDValue V = expandMulToAddOrSubOfShl(N, DAG, MulAmt))
    return V;
 
  if (!Subtarget.hasStdExtZmmul())
    return expandMulToNAFSequence(N, DAG, MulAmt);
 
  return SDValue();
}
 
// Combine vXi32 (mul (and (lshr X, 15), 0x10001), 0xffff) ->
// (bitcast (sra (v2Xi16 (bitcast X)), 15))
// Same for other equivalent types with other equivalent constants.
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
  // Do this for legal vectors unless they are i1 or i8 vectors.
  if (!VT.isVector() || !TLI.isTypeLegal(VT) || VT.getScalarSizeInBits() < 16)
    return SDValue();
 
  if (N->getOperand(0).getOpcode() != ISD::AND ||
      N->getOperand(0).getOperand(0).getOpcode() != ISD::SRL)
    return SDValue();
 
  SDValue And = N->getOperand(0);
  SDValue Srl = And.getOperand(0);
 
  APInt V1, V2, V3;
  if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), V1) ||
      !ISD::isConstantSplatVector(And.getOperand(1).getNode(), V2) ||
      !ISD::isConstantSplatVector(Srl.getOperand(1).getNode(), V3))
    return SDValue();
 
  unsigned HalfSize = VT.getScalarSizeInBits() / 2;
  if (!V1.isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
      V3 != (HalfSize - 1))
    return SDValue();
 
  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(),
                                EVT::getIntegerVT(*DAG.getContext(), HalfSize),
                                VT.getVectorElementCount() * 2);
  SDLoc DL(N);
  SDValue Cast = DAG.getNode(ISD::BITCAST, DL, HalfVT, Srl.getOperand(0));
  SDValue Sra = DAG.getNode(ISD::SRA, DL, HalfVT, Cast,
                            DAG.getConstant(HalfSize - 1, DL, HalfVT));
  return DAG.getNode(ISD::BITCAST, DL, VT, Sra);
}
 
static SDValue combinePExtWideningMul(SDNode *N, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  if (!Subtarget.hasStdExtP())
    return SDValue();
 
  EVT VT = N->getValueType(0);
  if (VT != MVT::v4i16 && VT != MVT::v2i32)
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  bool N0IsSExt = N0.getOpcode() == ISD::SIGN_EXTEND;
  bool N0IsZExt = N0.getOpcode() == ISD::ZERO_EXTEND;
  bool N1IsSExt = N1.getOpcode() == ISD::SIGN_EXTEND;
  bool N1IsZExt = N1.getOpcode() == ISD::ZERO_EXTEND;
 
  if (!(N0IsSExt || N0IsZExt) || !(N1IsSExt || N1IsZExt) || !N0.hasOneUse() ||
      !N1.hasOneUse())
    return SDValue();
 
  SDValue A = N0.getOperand(0);
  SDValue B = N1.getOperand(0);
  EVT SrcVT = VT == MVT::v4i16 ? MVT::v4i8 : MVT::v2i16;
  if (A.getValueType() != SrcVT || B.getValueType() != SrcVT)
    return SDValue();
 
  unsigned RV32Opc, RV64Opc;
  bool IsSignedUnsigned = false;
  if (N0IsSExt && N1IsSExt) {
    RV32Opc = RISCVISD::PWMUL;
    RV64Opc = RISCVISD::PMUL_HALVES_01;
  } else if (N0IsZExt && N1IsZExt) {
    RV32Opc = RISCVISD::PWMULU;
    RV64Opc = RISCVISD::PMULU_HALVES_01;
  } else {
    IsSignedUnsigned = true;
    RV32Opc = RISCVISD::PWMULSU;
    RV64Opc = RISCVISD::PMULSU_HALVES_00;
    if (N0IsZExt && N1IsSExt)
      std::swap(A, B);
  }
 
  SDLoc DL(N);
  if (!Subtarget.is64Bit())
    return DAG.getNode(RV32Opc, DL, VT, A, B);
 
  MVT LegalSrcVT = VT == MVT::v4i16 ? MVT::v8i8 : MVT::v4i16;
  A = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, A, DAG.getUNDEF(SrcVT));
  B = DAG.getNode(ISD::CONCAT_VECTORS, DL, LegalSrcVT, B, DAG.getUNDEF(SrcVT));
 
  if (IsSignedUnsigned) {
    SDValue Zero = DAG.getConstant(0, DL, LegalSrcVT);
    A = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, A, Zero);
    B = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, B, Zero);
    return DAG.getNode(RV64Opc, DL, VT, A, B);
  }
 
  SDValue Zip = DAG.getNode(RISCVISD::PZIP, DL, LegalSrcVT, A, B);
  return DAG.getNode(RV64Opc, DL, VT, Zip, Zip);
}
 
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
                                 TargetLowering::DAGCombinerInfo &DCI,
                                 const RISCVSubtarget &Subtarget) {
  EVT VT = N->getValueType(0);
  if (!VT.isVector())
    return expandMul(N, DAG, DCI, Subtarget);
 
  SDLoc DL(N);
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue MulOper;
  unsigned AddSubOpc;
 
  // vmadd: (mul (add x, 1), y) -> (add (mul x, y), y)
  //        (mul x, add (y, 1)) -> (add x, (mul x, y))
  // vnmsub: (mul (sub 1, x), y) -> (sub y, (mul x, y))
  //         (mul x, (sub 1, y)) -> (sub x, (mul x, y))
  auto IsAddSubWith1 = [&](SDValue V) -> bool {
    AddSubOpc = V->getOpcode();
    if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
      SDValue Opnd = V->getOperand(1);
      MulOper = V->getOperand(0);
      if (AddSubOpc == ISD::SUB)
        std::swap(Opnd, MulOper);
      if (isOneOrOneSplat(Opnd))
        return true;
    }
    return false;
  };
 
  if (IsAddSubWith1(N0)) {
    SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
    return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal);
  }
 
  if (IsAddSubWith1(N1)) {
    SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
    return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
  }
 
  if (SDValue V = combineBinOpOfZExt(N, DAG))
    return V;
 
  if (SDValue V = combinePExtWideningMul(N, DAG, Subtarget))
    return V;
 
  if (SDValue V = combineVectorMulToSraBitcast(N, DAG))
    return V;
 
  return SDValue();
}
 
/// According to the property that indexed load/store instructions zero-extend
/// their indices, try to narrow the type of index operand.
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
  if (isIndexTypeSigned(IndexType))
    return false;
 
  if (!N->hasOneUse())
    return false;
 
  EVT VT = N.getValueType();
  SDLoc DL(N);
 
  // In general, what we're doing here is seeing if we can sink a truncate to
  // a smaller element type into the expression tree building our index.
  // TODO: We can generalize this and handle a bunch more cases if useful.
 
  // Narrow a buildvector to the narrowest element type.  This requires less
  // work and less register pressure at high LMUL, and creates smaller constants
  // which may be cheaper to materialize.
  if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) {
    KnownBits Known = DAG.computeKnownBits(N);
    unsigned ActiveBits = std::max(8u, Known.countMaxActiveBits());
    LLVMContext &C = *DAG.getContext();
    EVT ResultVT = EVT::getIntegerVT(C, ActiveBits).getRoundIntegerType(C);
    if (ResultVT.bitsLT(VT.getVectorElementType())) {
      N = DAG.getNode(ISD::TRUNCATE, DL,
                      VT.changeVectorElementType(C, ResultVT), N);
      return true;
    }
  }
 
  // Handle the pattern (shl (zext x to ty), C) and bits(x) + C < bits(ty).
  if (N.getOpcode() != ISD::SHL)
    return false;
 
  SDValue N0 = N.getOperand(0);
  if (N0.getOpcode() != ISD::ZERO_EXTEND &&
      N0.getOpcode() != RISCVISD::VZEXT_VL)
    return false;
  if (!N0->hasOneUse())
    return false;
 
  APInt ShAmt;
  SDValue N1 = N.getOperand(1);
  if (!ISD::isConstantSplatVector(N1.getNode(), ShAmt))
    return false;
 
  SDValue Src = N0.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned SrcElen = SrcVT.getScalarSizeInBits();
 
  // Consider any leading zeros in the source.
  SrcElen -= DAG.computeKnownBits(Src).countMinLeadingZeros();
 
  unsigned ShAmtV = ShAmt.getZExtValue();
  unsigned NewElen = PowerOf2Ceil(SrcElen + ShAmtV);
  NewElen = std::max(NewElen, 8U);
  // Make sure the new elen is at least as large as the original elen.
  NewElen = std::max<unsigned>(NewElen, SrcVT.getScalarSizeInBits());
 
  // Skip if NewElen is not narrower than the original extended type.
  if (NewElen >= N0.getValueType().getScalarSizeInBits())
    return false;
 
  EVT NewEltVT = EVT::getIntegerVT(*DAG.getContext(), NewElen);
  EVT NewVT = SrcVT.changeVectorElementType(*DAG.getContext(), NewEltVT);
 
  SDValue NewExt = DAG.getNode(N0->getOpcode(), DL, NewVT, N0->ops());
  SDValue NewShAmtVec = DAG.getConstant(ShAmtV, DL, NewVT);
  N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec);
  return true;
}
 
/// Given
/// ```
/// %b = splat %base
/// %s = <%offset, %offset + 1, %offset + 2, %offset + 3, ...>
/// %a = add nuw %b, %s
/// %N = splat %n
/// %p = setcc ult %a, %N
/// ```
/// in which the SETCC's condition code could be (U)LT or (U)GT.
/// We can turn it into
/// ```
/// %s = <0, 1, 2, 3, ...>
/// %m = sub %n, min(%n, %base + %offset)
/// %M = splat %m
/// %p = setcc ult %s, %M
/// ```
/// where the SETCC's condition code is also canonicalized into (U)LT;
/// The idea behind this canonicalization is that if %p is used as a mask
/// in a masked.load/store, we can easily turn it to use VL-predicate later
/// by assigning `vl = min(%m, <number of vector elements>)`.
static SDValue canonicalizeMaskForVLPredicate(
    EVT MaskVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL,
    SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI) {
 
  // This transformation performs checks against the vector element type, which
  // is also used to generate scalar value that will be splatted later. Because
  // of this, the emitted scalar value might not be legal type and therefore we
  // need to run this before type legalizer.
  if (!DCI.isBeforeLegalize())
    return SDValue();
 
  using namespace SDPatternMatch;
  if (!MaskVT.isFixedLengthVector() ||
      !(CC == ISD::SETUGT || CC == ISD::SETGT || CC == ISD::SETULT ||
        CC == ISD::SETLT) ||
      !LHS.getValueType().isInteger())
    return SDValue();
 
  // Canonicalize the comparison.
  if (CC == ISD::SETUGT || CC == ISD::SETGT) {
    std::swap(LHS, RHS);
    CC = ISD::getSetCCSwappedOperands(CC);
  }
  bool IsSigned = ISD::isSignedIntSetCC(CC);
 
  EVT ElementVT = LHS.getValueType().getScalarType();
  uint64_t ElementSize = LHS.getScalarValueSizeInBits();
 
  SDValue Boundary = DAG.getSplatValue(RHS);
  if (!Boundary)
    return SDValue();
 
  SDValue LHSOp0, LHSOp1;
  if (IsSigned) {
    if (!sd_match(LHS, m_NSWAddLike(m_Value(LHSOp0), m_Value(LHSOp1))))
      return SDValue();
  } else {
    if (!sd_match(LHS, m_NUWAddLike(m_Value(LHSOp0), m_Value(LHSOp1))))
      return SDValue();
  }
 
  SDValue BaseIndex = DAG.getSplatValue(LHSOp0);
  if (!BaseIndex) {
    std::swap(LHSOp0, LHSOp1);
    BaseIndex = DAG.getSplatValue(LHSOp0);
  }
  if (!BaseIndex || !isa<BuildVectorSDNode>(LHSOp1))
    return SDValue();
 
  // Return {a,n} from a build_vector sequence of {a, a+n, a+2n, a+3n, ....}
  auto StepVector = cast<BuildVectorSDNode>(LHSOp1)->isArithmeticSequence();
  if (!StepVector || !StepVector->second.isOne())
    return SDValue();
  const APInt &Start = StepVector->first;
 
  Boundary = DAG.getExtOrTrunc(IsSigned, Boundary, DL, ElementVT);
  BaseIndex = DAG.getExtOrTrunc(IsSigned, BaseIndex, DL, ElementVT);
  SDValue Offset = DAG.getConstant(IsSigned ? Start.sextOrTrunc(ElementSize)
                                            : Start.zextOrTrunc(ElementSize),
                                   DL, ElementVT);
 
  if (IsSigned) {
    // Two additional conditions:
    // 1. Offset + BaseIndex never overflow
    if (!DAG.willNotOverflowAdd(/*IsSigned=*/true, BaseIndex, Offset))
      return SDValue();
 
    // 2. Offset + BaseIndex has to be non-negative
    auto BaseIndexKB = DAG.computeKnownBits(BaseIndex);
    auto OffsetKB = DAG.computeKnownBits(Offset);
    if (!KnownBits::add(BaseIndexKB, OffsetKB, /*NSW=*/true,
                        /*NUW=*/false)
             .isNonNegative())
      return SDValue();
  }
 
  unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
  SDValue NewStepVector = DAG.getStepVector(DL, LHS.getValueType());
  BaseIndex = DAG.getNode(ISD::ADD, DL, ElementVT, BaseIndex, Offset,
                          IsSigned ? SDNodeFlags::NoSignedWrap
                                   : SDNodeFlags::NoUnsignedWrap);
  BaseIndex = DAG.getNode(MinOpc, DL, ElementVT, Boundary, BaseIndex);
  Boundary = DAG.getNode(ISD::SUB, DL, ElementVT, Boundary, BaseIndex);
  Boundary = DAG.getSplat(RHS.getValueType(), DL, Boundary);
 
  return DAG.getSetCC(DL, MaskVT, NewStepVector, Boundary, CC);
}
 
/// Try to map an integer comparison with size > XLEN to vector instructions
/// before type legalization splits it up into chunks.
static SDValue
combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
                                const SDLoc &DL, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  assert(ISD::isIntEqualitySetCC(CC) && "Bad comparison predicate");
 
  if (!Subtarget.useRVVForFixedLengthVectors())
    return SDValue();
 
  MVT XLenVT = Subtarget.getXLenVT();
  EVT OpVT = X.getValueType();
  // We're looking for an oversized integer equality comparison.
  if (!OpVT.isScalarInteger())
    return SDValue();
 
  unsigned OpSize = OpVT.getSizeInBits();
  // The size should be larger than XLen and smaller than the maximum vector
  // size.
  if (OpSize <= Subtarget.getXLen() || OpSize > Subtarget.getRealMinVLen() * 8)
    return SDValue();
 
  // Don't perform this combine if constructing the vector will be expensive.
  auto IsVectorBitCastCheap = [](SDValue X) {
    X = peekThroughBitcasts(X);
    return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
           X.getOpcode() == ISD::LOAD;
  };
  if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
    return SDValue();
 
  if (DAG.getMachineFunction().getFunction().hasFnAttribute(
          Attribute::NoImplicitFloat))
    return SDValue();
 
  // Bail out for non-byte-sized types.
  if (!OpVT.isByteSized())
    return SDValue();
 
  // Find a preferred vector element type by inspecting how the value is used.
  auto GetPreferredEltVT = [](SDValue V) -> MVT {
    // Look backwards: check if V itself is derived from a vector
    SDValue Peek = peekThroughBitcasts(V);
    EVT PeekVT = Peek.getValueType();
 
    if (PeekVT.isVector() && PeekVT.isInteger()) {
      EVT EltVT = PeekVT.getVectorElementType();
      if (EltVT.isSimple())
        return EltVT.getSimpleVT();
    }
 
    // Look forwards: check if V is bitcasted to a vector elsewhere in the DAG
    for (SDUse &Use : V->uses()) {
      // Ensure we are checking the use of the specific value result, not the
      // node's chain
      if (Use.getResNo() != V.getResNo())
        continue;
 
      SDNode *User = Use.getUser();
      if (User->getOpcode() == ISD::BITCAST) {
        EVT CastVT = User->getValueType(0);
        if (CastVT.isVector() && CastVT.isInteger()) {
          EVT EltVT = CastVT.getVectorElementType();
          if (EltVT.isSimple())
            return EltVT.getSimpleVT();
        }
      }
    }
    return MVT::INVALID_SIMPLE_VALUE_TYPE;
  };
 
  auto IsValidEltVT = [&](MVT VT) {
    if (VT == MVT::INVALID_SIMPLE_VALUE_TYPE || !VT.isInteger())
      return false;
 
    // Make sure we don't try to create an impossible vector type where the
    // elements don't perfectly fill up the OpSize boundary.
    unsigned EltSize = VT.getSizeInBits();
    if (OpSize % EltSize != 0)
      return false;
 
    // Construct the proposed vector type to check its legality
    unsigned NumElts = OpSize / EltSize;
    EVT TestVecVT = EVT::getVectorVT(*DAG.getContext(), VT, NumElts);
    return Subtarget.getTargetLowering()->isTypeLegal(TestVecVT);
  };
 
  // Get preferred VT from either X or Y.
  MVT EltVT = GetPreferredEltVT(X);
  if (!IsValidEltVT(EltVT))
    EltVT = GetPreferredEltVT(Y);
 
  // If both are unsuitable, use the safe default (i8)
  if (!IsValidEltVT(EltVT))
    EltVT = MVT::i8;
 
  unsigned EltSize = EltVT.getSizeInBits();
  unsigned NumElts = OpSize / EltSize;
  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
  EVT CmpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
 
  SDValue VecX = DAG.getBitcast(VecVT, X);
  SDValue VecY;
  // Constant fold the common case of comparing with zero. Later optimizations
  // might not do this for us.
  if (isNullConstant(Y))
    VecY = DAG.getConstant(0, DL, VecVT);
  else
    VecY = DAG.getBitcast(VecVT, Y);
 
  SDValue Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
  return DAG.getSetCC(DL, VT, DAG.getNode(ISD::VECREDUCE_OR, DL, XLenVT, Cmp),
                      DAG.getConstant(0, DL, XLenVT), CC);
}
 
static SDValue performSETCCCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI,
                                   const RISCVSubtarget &Subtarget) {
  SelectionDAG &DAG = DCI.DAG;
  SDLoc dl(N);
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT OpVT = N0.getValueType();
 
  ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
  if (SDValue V =
          canonicalizeMaskForVLPredicate(VT, N0, N1, Cond, dl, DAG, DCI))
    return V;
 
  // Looking for an equality compare.
  if (!isIntEqualitySetCC(Cond))
    return SDValue();
 
  if (SDValue V =
          combineVectorSizedSetCCEquality(VT, N0, N1, Cond, dl, DAG, Subtarget))
    return V;
 
  if (DCI.isAfterLegalizeDAG() && isa<ConstantSDNode>(N1) &&
      N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
      isa<ConstantSDNode>(N0.getOperand(1))) {
    const APInt &AndRHSC = N0.getConstantOperandAPInt(1);
    // (X & -(1 << C)) == 0 -> (X >> C) == 0 if the AND constant can't use ANDI.
    if (isNullConstant(N1) && !isInt<12>(AndRHSC.getSExtValue()) &&
        AndRHSC.isNegatedPowerOf2()) {
      unsigned ShiftBits = AndRHSC.countr_zero();
      SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, N0.getOperand(0),
                                  DAG.getConstant(ShiftBits, dl, OpVT));
      return DAG.getSetCC(dl, VT, Shift, N1, Cond);
    }
 
    // Similar to above but handling the lower 32 bits by using sraiw. Allow
    // comparing with constants other than 0 if the constant can be folded into
    // addi or xori after shifting.
    uint64_t N1Int = cast<ConstantSDNode>(N1)->getZExtValue();
    uint64_t AndRHSInt = AndRHSC.getZExtValue();
    if (OpVT == MVT::i64 && isUInt<32>(AndRHSInt) &&
        isPowerOf2_32(-uint32_t(AndRHSInt)) && (N1Int & AndRHSInt) == N1Int) {
      unsigned ShiftBits = llvm::countr_zero(AndRHSInt);
      int64_t NewC = SignExtend64<32>(N1Int) >> ShiftBits;
      if (ShiftBits != 0 && NewC >= -2048 && NewC <= 2048) {
        SDValue SExt =
            DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OpVT, N0.getOperand(0),
                        DAG.getValueType(MVT::i32));
        SDValue Shift = DAG.getNode(ISD::SRA, dl, OpVT, SExt,
                                    DAG.getConstant(ShiftBits, dl, OpVT));
        return DAG.getSetCC(dl, VT, Shift,
                            DAG.getSignedConstant(NewC, dl, OpVT), Cond);
      }
    }
 
    // Fold (and X, Mask) ==/!= C -> X ==/!= sext(C, countr_one(Mask)) if the
    // Mask is only clearing redundant sign bits.
    if (isMask_64(AndRHSInt)) {
      unsigned TrailingOnes = llvm::countr_one(AndRHSInt);
      unsigned N1Width = llvm::bit_width(N1Int);
      int64_t N1SExt = SignExtend64(N1Int, TrailingOnes);
      if (N1Width <= TrailingOnes && isInt<12>(N1SExt) &&
          DAG.ComputeMaxSignificantBits(N0.getOperand(0)) <= TrailingOnes)
        return DAG.getSetCC(dl, VT, N0.getOperand(0),
                            DAG.getSignedConstant(N1SExt, dl, OpVT), Cond);
    }
  }
 
  // Replace (seteq (i64 (and X, 0xffffffff)), C1) with
  // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
  // bit 31. Same for setne. C1' may be cheaper to materialize and the
  // sext_inreg can become a sext.w instead of a shift pair.
  if (OpVT != MVT::i64 || !Subtarget.is64Bit())
    return SDValue();
 
  // RHS needs to be a constant.
  auto *N1C = dyn_cast<ConstantSDNode>(N1);
  if (!N1C)
    return SDValue();
 
  // LHS needs to be (and X, 0xffffffff).
  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
      !isa<ConstantSDNode>(N0.getOperand(1)) ||
      N0.getConstantOperandVal(1) != UINT64_C(0xffffffff))
    return SDValue();
 
  // Don't do this if the sign bit is provably zero, it will be turned back into
  // an AND.
  APInt SignMask = APInt::getOneBitSet(64, 31);
  if (DAG.MaskedValueIsZero(N0.getOperand(0), SignMask))
    return SDValue();
 
  const APInt &C1 = N1C->getAPIntValue();
 
  // If the constant is larger than 2^32 - 1 it is impossible for both sides
  // to be equal.
  if (C1.getActiveBits() > 32)
    return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
 
  SDValue SExtOp = DAG.getNode(ISD::SIGN_EXTEND_INREG, N, OpVT,
                               N0.getOperand(0), DAG.getValueType(MVT::i32));
  return DAG.getSetCC(dl, VT, SExtOp, DAG.getConstant(C1.trunc(32).sext(64),
                                                      dl, OpVT), Cond);
}
 
static SDValue
performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                const RISCVSubtarget &Subtarget) {
  SelectionDAG &DAG = DCI.DAG;
  SDValue Src = N->getOperand(0);
  EVT VT = N->getValueType(0);
  EVT SrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
  unsigned Opc = Src.getOpcode();
  SDLoc DL(N);
 
  // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
  // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
  if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(MVT::i16) &&
      Subtarget.hasStdExtZfhmin())
    return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, DL, VT, Src.getOperand(0));
 
  // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
  if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
      VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
      DAG.computeKnownBits(Src.getOperand(1)).countMaxActiveBits() <= 5)
    return DAG.getNode(RISCVISD::SLLW, DL, VT, Src.getOperand(0),
                       Src.getOperand(1));
 
  // Fold (sext_inreg (setcc), i1) -> (sub 0, (setcc))
  if (Opc == ISD::SETCC && SrcVT == MVT::i1 && DCI.isAfterLegalizeDAG())
    return DAG.getNegative(Src, DL, VT);
 
  // Fold (sext_inreg (xor (setcc), -1), i1) -> (add (setcc), -1)
  if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
      isAllOnesConstant(Src.getOperand(1)) &&
      Src.getOperand(0).getOpcode() == ISD::SETCC && DCI.isAfterLegalizeDAG())
    return DAG.getNode(ISD::ADD, DL, VT, Src.getOperand(0),
                       DAG.getAllOnesConstant(DL, VT));
 
  return SDValue();
}
 
namespace {
// Forward declaration of the structure holding the necessary information to
// apply a combine.
struct CombineResult;
 
enum ExtKind : uint8_t {
  ZExt = 1 << 0,
  SExt = 1 << 1,
  FPExt = 1 << 2,
  BF16Ext = 1 << 3
};
/// Helper class for folding sign/zero extensions.
/// In particular, this class is used for the following combines:
/// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
/// mul | mul_vl -> vwmul(u) | vwmul_su
/// shl | shl_vl -> vwsll
/// fadd -> vfwadd | vfwadd_w