/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Bug Summary

File:	llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1110, column 10 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name RISCVISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/build-llvm/lib/Target/RISCV -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/build-llvm/lib/Target/RISCV -I /build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV -I /build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/build-llvm/lib/Target/RISCV -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-03-10-135947-24842-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp

/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp

→

1//===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISCV uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//

14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVRegisterInfo.h"
19#include "RISCVSubtarget.h"
20#include "RISCVTargetMachine.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/CodeGen/CallingConvLower.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
29#include "llvm/CodeGen/ValueTypes.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/DiagnosticPrinter.h"
32#include "llvm/IR/IntrinsicsRISCV.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Support/raw_ostream.h"

39using namespace llvm;

41#define DEBUG_TYPE"riscv-lower" "riscv-lower"

43STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"riscv-lower", "NumTailCalls"
, "Number of tail calls"};

45RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                                       const RISCVSubtarget &STI)
  : TargetLowering(TM), Subtarget(STI) {

if (Subtarget.isRV32E())
  report_fatal_error("Codegen not yet implemented for RV32E");

RISCVABI::ABI ABI = Subtarget.getTargetABI();
assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI")((ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI"
) ? static_cast<void> (0) : __assert_fail ("ABI != RISCVABI::ABI_Unknown && \"Improperly initialised target ABI\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 53, __PRETTY_FUNCTION__));

if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
    !Subtarget.hasStdExtF()) {
  errs() << "Hard-float 'f' ABI can't be used for a target that "
              "doesn't support the F instruction set extension (ignoring "
                        "target-abi)\n";
  ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
} else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
           !Subtarget.hasStdExtD()) {
  errs() << "Hard-float 'd' ABI can't be used for a target that "
            "doesn't support the D instruction set extension (ignoring "
            "target-abi)\n";
  ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
}

switch (ABI) {
default:
  report_fatal_error("Don't know how to lower this ABI");
case RISCVABI::ABI_ILP32:
case RISCVABI::ABI_ILP32F:
case RISCVABI::ABI_ILP32D:
case RISCVABI::ABI_LP64:
case RISCVABI::ABI_LP64F:
case RISCVABI::ABI_LP64D:
  break;
}

MVT XLenVT = Subtarget.getXLenVT();

// Set up the register classes.
addRegisterClass(XLenVT, &RISCV::GPRRegClass);

if (Subtarget.hasStdExtZfh())
  addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
if (Subtarget.hasStdExtF())
  addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
if (Subtarget.hasStdExtD())
  addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);

static const MVT::SimpleValueType BoolVecVTs[] = {
    MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
    MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
static const MVT::SimpleValueType IntVecVTs[] = {
    MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
    MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
    MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
    MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
    MVT::nxv4i64, MVT::nxv8i64};
static const MVT::SimpleValueType F16VecVTs[] = {
    MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
    MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
static const MVT::SimpleValueType F32VecVTs[] = {
    MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
static const MVT::SimpleValueType F64VecVTs[] = {
    MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};

if (Subtarget.hasStdExtV()) {
  auto addRegClassForRVV = [this](MVT VT) {
    unsigned Size = VT.getSizeInBits().getKnownMinValue();
    assert(Size <= 512 && isPowerOf2_32(Size))((Size <= 512 && isPowerOf2_32(Size)) ? static_cast
<void> (0) : __assert_fail ("Size <= 512 && isPowerOf2_32(Size)"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 113, __PRETTY_FUNCTION__));
    const TargetRegisterClass *RC;
    if (Size <= 64)
      RC = &RISCV::VRRegClass;
    else if (Size == 128)
      RC = &RISCV::VRM2RegClass;
    else if (Size == 256)
      RC = &RISCV::VRM4RegClass;
    else
      RC = &RISCV::VRM8RegClass;

    addRegisterClass(VT, RC);
  };

  for (MVT VT : BoolVecVTs)
    addRegClassForRVV(VT);
  for (MVT VT : IntVecVTs)
    addRegClassForRVV(VT);

  if (Subtarget.hasStdExtZfh())
    for (MVT VT : F16VecVTs)
      addRegClassForRVV(VT);

  if (Subtarget.hasStdExtF())
    for (MVT VT : F32VecVTs)
      addRegClassForRVV(VT);

  if (Subtarget.hasStdExtD())
    for (MVT VT : F64VecVTs)
      addRegClassForRVV(VT);

  if (Subtarget.useRVVForFixedLengthVectors()) {
    auto addRegClassForFixedVectors = [this](MVT VT) {
      unsigned LMul = Subtarget.getLMULForFixedLengthVector(VT);
      const TargetRegisterClass *RC;
      if (LMul == 1)
        RC = &RISCV::VRRegClass;
      else if (LMul == 2)
        RC = &RISCV::VRM2RegClass;
      else if (LMul == 4)
        RC = &RISCV::VRM4RegClass;
      else if (LMul == 8)
        RC = &RISCV::VRM8RegClass;
      else
        llvm_unreachable("Unexpected LMul!")::llvm::llvm_unreachable_internal("Unexpected LMul!", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 157);

      addRegisterClass(VT, RC);
    };
    for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
      if (useRVVForFixedLengthVectorVT(VT))
        addRegClassForFixedVectors(VT);

    for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
      if (useRVVForFixedLengthVectorVT(VT))
        addRegClassForFixedVectors(VT);
  }
}

// Compute derived properties from the register classes.
computeRegisterProperties(STI.getRegisterInfo());

setStackPointerRegisterToSaveRestore(RISCV::X2);

for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
  setLoadExtAction(N, XLenVT, MVT::i1, Promote);

// TODO: add all necessary setOperationAction calls.
setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);

setOperationAction(ISD::BR_JT, MVT::Other, Expand);
setOperationAction(ISD::BR_CC, XLenVT, Expand);
setOperationAction(ISD::SELECT_CC, XLenVT, Expand);

setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);

setOperationAction(ISD::VASTART, MVT::Other, Custom);
setOperationAction(ISD::VAARG, MVT::Other, Expand);
setOperationAction(ISD::VACOPY, MVT::Other, Expand);
setOperationAction(ISD::VAEND, MVT::Other, Expand);

setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
if (!Subtarget.hasStdExtZbb()) {
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
}

if (Subtarget.is64Bit()) {
  setOperationAction(ISD::ADD, MVT::i32, Custom);
  setOperationAction(ISD::SUB, MVT::i32, Custom);
  setOperationAction(ISD::SHL, MVT::i32, Custom);
  setOperationAction(ISD::SRA, MVT::i32, Custom);
  setOperationAction(ISD::SRL, MVT::i32, Custom);
}

if (!Subtarget.hasStdExtM()) {
  setOperationAction(ISD::MUL, XLenVT, Expand);
  setOperationAction(ISD::MULHS, XLenVT, Expand);
  setOperationAction(ISD::MULHU, XLenVT, Expand);
  setOperationAction(ISD::SDIV, XLenVT, Expand);
  setOperationAction(ISD::UDIV, XLenVT, Expand);
  setOperationAction(ISD::SREM, XLenVT, Expand);
  setOperationAction(ISD::UREM, XLenVT, Expand);
}

if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) {
  setOperationAction(ISD::MUL, MVT::i32, Custom);

  setOperationAction(ISD::SDIV, MVT::i8, Custom);
  setOperationAction(ISD::UDIV, MVT::i8, Custom);
  setOperationAction(ISD::UREM, MVT::i8, Custom);
  setOperationAction(ISD::SDIV, MVT::i16, Custom);
  setOperationAction(ISD::UDIV, MVT::i16, Custom);
  setOperationAction(ISD::UREM, MVT::i16, Custom);
  setOperationAction(ISD::SDIV, MVT::i32, Custom);
  setOperationAction(ISD::UDIV, MVT::i32, Custom);
  setOperationAction(ISD::UREM, MVT::i32, Custom);
}

setOperationAction(ISD::SDIVREM, XLenVT, Expand);
setOperationAction(ISD::UDIVREM, XLenVT, Expand);
setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);

setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);

if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::ROTL, MVT::i32, Custom);
    setOperationAction(ISD::ROTR, MVT::i32, Custom);
  }
} else {
  setOperationAction(ISD::ROTL, XLenVT, Expand);
  setOperationAction(ISD::ROTR, XLenVT, Expand);
}

if (Subtarget.hasStdExtZbp()) {
  // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
  // more combining.
  setOperationAction(ISD::BITREVERSE, XLenVT, Custom);
  setOperationAction(ISD::BSWAP, XLenVT, Custom);

  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
    setOperationAction(ISD::BSWAP, MVT::i32, Custom);
  }
} else {
  // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
  // pattern match it directly in isel.
  setOperationAction(ISD::BSWAP, XLenVT,
                     Subtarget.hasStdExtZbb() ? Legal : Expand);
}

if (Subtarget.hasStdExtZbb()) {
  setOperationAction(ISD::SMIN, XLenVT, Legal);
  setOperationAction(ISD::SMAX, XLenVT, Legal);
  setOperationAction(ISD::UMIN, XLenVT, Legal);
  setOperationAction(ISD::UMAX, XLenVT, Legal);
} else {
  setOperationAction(ISD::CTTZ, XLenVT, Expand);
  setOperationAction(ISD::CTLZ, XLenVT, Expand);
  setOperationAction(ISD::CTPOP, XLenVT, Expand);
}

if (Subtarget.hasStdExtZbt()) {
  setOperationAction(ISD::FSHL, XLenVT, Custom);
  setOperationAction(ISD::FSHR, XLenVT, Custom);
  setOperationAction(ISD::SELECT, XLenVT, Legal);

  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::FSHL, MVT::i32, Custom);
    setOperationAction(ISD::FSHR, MVT::i32, Custom);
  }
} else {
  setOperationAction(ISD::SELECT, XLenVT, Custom);
}

ISD::CondCode FPCCToExpand[] = {
    ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
    ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
    ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};

ISD::NodeType FPOpToExpand[] = {
    ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FP16_TO_FP,
    ISD::FP_TO_FP16};

if (Subtarget.hasStdExtZfh())
  setOperationAction(ISD::BITCAST, MVT::i16, Custom);

if (Subtarget.hasStdExtZfh()) {
  setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
  setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
  for (auto CC : FPCCToExpand)
    setCondCodeAction(CC, MVT::f16, Expand);
  setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
  setOperationAction(ISD::SELECT, MVT::f16, Custom);
  setOperationAction(ISD::BR_CC, MVT::f16, Expand);
  for (auto Op : FPOpToExpand)
    setOperationAction(Op, MVT::f16, Expand);
}

if (Subtarget.hasStdExtF()) {
  setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
  setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
  for (auto CC : FPCCToExpand)
    setCondCodeAction(CC, MVT::f32, Expand);
  setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
  setOperationAction(ISD::SELECT, MVT::f32, Custom);
  setOperationAction(ISD::BR_CC, MVT::f32, Expand);
  for (auto Op : FPOpToExpand)
    setOperationAction(Op, MVT::f32, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
}

if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
  setOperationAction(ISD::BITCAST, MVT::i32, Custom);

if (Subtarget.hasStdExtD()) {
  setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
  setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
  for (auto CC : FPCCToExpand)
    setCondCodeAction(CC, MVT::f64, Expand);
  setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
  setOperationAction(ISD::SELECT, MVT::f64, Custom);
  setOperationAction(ISD::BR_CC, MVT::f64, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
  for (auto Op : FPOpToExpand)
    setOperationAction(Op, MVT::f64, Expand);
  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
}

if (Subtarget.is64Bit()) {
  setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
}

setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
setOperationAction(ISD::BlockAddress, XLenVT, Custom);
setOperationAction(ISD::ConstantPool, XLenVT, Custom);
setOperationAction(ISD::JumpTable, XLenVT, Custom);

setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);

// TODO: On M-mode only targets, the cycle[h] CSR may not be present.
// Unfortunately this can't be determined just from the ISA naming string.
setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
                   Subtarget.is64Bit() ? Legal : Custom);

setOperationAction(ISD::TRAP, MVT::Other, Legal);
setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);

if (Subtarget.hasStdExtA()) {
  setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
  setMinCmpXchgSizeInBits(32);
} else {
  setMaxAtomicSizeInBitsSupported(0);
}

setBooleanContents(ZeroOrOneBooleanContent);

if (Subtarget.hasStdExtV()) {
  setBooleanVectorContents(ZeroOrOneBooleanContent);

  setOperationAction(ISD::VSCALE, XLenVT, Custom);

  // RVV intrinsics may have illegal operands.
  // We also need to custom legalize vmv.x.s.
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);

  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);

  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
  } else {
    // We must custom-lower certain vXi64 operations on RV32 due to the vector
    // element type being illegal.
    setOperationAction(ISD::SPLAT_VECTOR, MVT::i64, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);

    setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
    setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
  }

  for (MVT VT : BoolVecVTs) {
    setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);

    // Mask VTs are custom-expanded into a series of standard nodes
    setOperationAction(ISD::TRUNCATE, VT, Custom);
    setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
  }

  for (MVT VT : IntVecVTs) {
    setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);

    setOperationAction(ISD::SMIN, VT, Legal);
    setOperationAction(ISD::SMAX, VT, Legal);
    setOperationAction(ISD::UMIN, VT, Legal);
    setOperationAction(ISD::UMAX, VT, Legal);

    if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64)
      setOperationAction(ISD::ABS, VT, Custom);

    setOperationAction(ISD::ROTL, VT, Expand);
    setOperationAction(ISD::ROTR, VT, Expand);

    // Custom-lower extensions and truncations from/to mask types.
    setOperationAction(ISD::ANY_EXTEND, VT, Custom);
    setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
    setOperationAction(ISD::ZERO_EXTEND, VT, Custom);

    // RVV has native int->float & float->int conversions where the
    // element type sizes are within one power-of-two of each other. Any
    // wider distances between type sizes have to be lowered as sequences
    // which progressively narrow the gap in stages.
    setOperationAction(ISD::SINT_TO_FP, VT, Custom);
    setOperationAction(ISD::UINT_TO_FP, VT, Custom);
    setOperationAction(ISD::FP_TO_SINT, VT, Custom);
    setOperationAction(ISD::FP_TO_UINT, VT, Custom);

    // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
    // nodes which truncate by one power of two at a time.
    setOperationAction(ISD::TRUNCATE, VT, Custom);

    // Custom-lower insert/extract operations to simplify patterns.
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);

    // Custom-lower reduction operations to set up the corresponding custom
    // nodes' operands.
    setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
    setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
    setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
    setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
    setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
    setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
    setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
    setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);

    setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);

    setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
  }

  // Expand various CCs to best match the RVV ISA, which natively supports UNE
  // but no other unordered comparisons, and supports all ordered comparisons
  // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
  // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
  // and we pattern-match those back to the "original", swapping operands once
  // more. This way we catch both operations and both "vf" and "fv" forms with
  // fewer patterns.
  ISD::CondCode VFPCCToExpand[] = {
      ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
      ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
      ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
  };

  // Sets common operation actions on RVV floating-point vector types.
  const auto SetCommonVFPActions = [&](MVT VT) {
    setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
    // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
    // sizes are within one power-of-two of each other. Therefore conversions
    // between vXf16 and vXf64 must be lowered as sequences which convert via
    // vXf32.
    setOperationAction(ISD::FP_ROUND, VT, Custom);
    setOperationAction(ISD::FP_EXTEND, VT, Custom);
    // Custom-lower insert/extract operations to simplify patterns.
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    // Expand various condition codes (explained above).
    for (auto CC : VFPCCToExpand)
      setCondCodeAction(CC, VT, Expand);

    setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
    setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
    setOperationAction(ISD::FCOPYSIGN, VT, Legal);

    setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);

    setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
  };

  if (Subtarget.hasStdExtZfh())
    for (MVT VT : F16VecVTs)
      SetCommonVFPActions(VT);

  if (Subtarget.hasStdExtF())
    for (MVT VT : F32VecVTs)
      SetCommonVFPActions(VT);

  if (Subtarget.hasStdExtD())
    for (MVT VT : F64VecVTs)
      SetCommonVFPActions(VT);

  if (Subtarget.useRVVForFixedLengthVectors()) {
    for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
      if (!useRVVForFixedLengthVectorVT(VT))
        continue;

      // By default everything must be expanded.
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
        setOperationAction(Op, VT, Expand);
      for (MVT OtherVT : MVT::fixedlen_vector_valuetypes())
        setTruncStoreAction(VT, OtherVT, Expand);

      // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);

      setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);

      setOperationAction(ISD::LOAD, VT, Custom);
      setOperationAction(ISD::STORE, VT, Custom);

      setOperationAction(ISD::SETCC, VT, Custom);

      setOperationAction(ISD::TRUNCATE, VT, Custom);

      // Operations below are different for between masks and other vectors.
      if (VT.getVectorElementType() == MVT::i1) {
        setOperationAction(ISD::AND, VT, Custom);
        setOperationAction(ISD::OR, VT, Custom);
        setOperationAction(ISD::XOR, VT, Custom);
        continue;
      }

      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);

      setOperationAction(ISD::ADD, VT, Custom);
      setOperationAction(ISD::MUL, VT, Custom);
      setOperationAction(ISD::SUB, VT, Custom);
      setOperationAction(ISD::AND, VT, Custom);
      setOperationAction(ISD::OR, VT, Custom);
      setOperationAction(ISD::XOR, VT, Custom);
      setOperationAction(ISD::SDIV, VT, Custom);
      setOperationAction(ISD::SREM, VT, Custom);
      setOperationAction(ISD::UDIV, VT, Custom);
      setOperationAction(ISD::UREM, VT, Custom);
      setOperationAction(ISD::SHL, VT, Custom);
      setOperationAction(ISD::SRA, VT, Custom);
      setOperationAction(ISD::SRL, VT, Custom);

      setOperationAction(ISD::SMIN, VT, Custom);
      setOperationAction(ISD::SMAX, VT, Custom);
      setOperationAction(ISD::UMIN, VT, Custom);
      setOperationAction(ISD::UMAX, VT, Custom);
      setOperationAction(ISD::ABS,  VT, Custom);

      setOperationAction(ISD::MULHS, VT, Custom);
      setOperationAction(ISD::MULHU, VT, Custom);

      setOperationAction(ISD::SINT_TO_FP, VT, Custom);
      setOperationAction(ISD::UINT_TO_FP, VT, Custom);
      setOperationAction(ISD::FP_TO_SINT, VT, Custom);
      setOperationAction(ISD::FP_TO_UINT, VT, Custom);

      setOperationAction(ISD::VSELECT, VT, Custom);

      setOperationAction(ISD::ANY_EXTEND, VT, Custom);
      setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
      setOperationAction(ISD::ZERO_EXTEND, VT, Custom);

      setOperationAction(ISD::BITCAST, VT, Custom);

      // Custom-lower reduction operations to set up the corresponding custom
      // nodes' operands.
      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
    }

    for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
      if (!useRVVForFixedLengthVectorVT(VT))
        continue;

      // By default everything must be expanded.
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
        setOperationAction(Op, VT, Expand);
      for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
        setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
        setTruncStoreAction(VT, OtherVT, Expand);
      }

      // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);

      setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);

      setOperationAction(ISD::LOAD, VT, Custom);
      setOperationAction(ISD::STORE, VT, Custom);
      setOperationAction(ISD::FADD, VT, Custom);
      setOperationAction(ISD::FSUB, VT, Custom);
      setOperationAction(ISD::FMUL, VT, Custom);
      setOperationAction(ISD::FDIV, VT, Custom);
      setOperationAction(ISD::FNEG, VT, Custom);
      setOperationAction(ISD::FABS, VT, Custom);
      setOperationAction(ISD::FSQRT, VT, Custom);
      setOperationAction(ISD::FMA, VT, Custom);

      setOperationAction(ISD::FP_ROUND, VT, Custom);
      setOperationAction(ISD::FP_EXTEND, VT, Custom);

      for (auto CC : VFPCCToExpand)
        setCondCodeAction(CC, VT, Expand);

      setOperationAction(ISD::VSELECT, VT, Custom);

      setOperationAction(ISD::BITCAST, VT, Custom);

      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
    }
  }
}

// Function alignments.
const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
setMinFunctionAlignment(FunctionAlignment);
setPrefFunctionAlignment(FunctionAlignment);

setMinimumJumpTableEntries(5);

// Jumps are expensive, compared to logic
setJumpIsExpensive();

// We can use any register for comparisons
setHasMultipleConditionRegisters();

setTargetDAGCombine(ISD::SETCC);
if (Subtarget.hasStdExtZbp()) {
  setTargetDAGCombine(ISD::OR);
}
if (Subtarget.hasStdExtV())
  setTargetDAGCombine(ISD::FCOPYSIGN);
682}

684EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
                                          LLVMContext &Context,
                                          EVT VT) const {
if (!VT.isVector())
  return getPointerTy(DL);
if (Subtarget.hasStdExtV() &&
    (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
  return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
return VT.changeVectorElementTypeToInteger();
693}

695bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                           const CallInst &I,
                                           MachineFunction &MF,
                                           unsigned Intrinsic) const {
switch (Intrinsic) {
default:
  return false;
case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
case Intrinsic::riscv_masked_atomicrmw_add_i32:
case Intrinsic::riscv_masked_atomicrmw_sub_i32:
case Intrinsic::riscv_masked_atomicrmw_nand_i32:
case Intrinsic::riscv_masked_atomicrmw_max_i32:
case Intrinsic::riscv_masked_atomicrmw_min_i32:
case Intrinsic::riscv_masked_atomicrmw_umax_i32:
case Intrinsic::riscv_masked_atomicrmw_umin_i32:
case Intrinsic::riscv_masked_cmpxchg_i32:
  PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::getVT(PtrTy->getElementType());
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(4);
  Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
               MachineMemOperand::MOVolatile;
  return true;
}
721}

723bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                              const AddrMode &AM, Type *Ty,
                                              unsigned AS,
                                              Instruction *I) const {
// No global is ever allowed as a base.
if (AM.BaseGV)
  return false;

// Require a 12-bit signed offset.
if (!isInt<12>(AM.BaseOffs))
  return false;

switch (AM.Scale) {
case 0: // "r+i" or just "i", depending on HasBaseReg.
  break;
case 1:
  if (!AM.HasBaseReg) // allow "r+i".
    break;
  return false; // disallow "r+r" or "r+r+i".
default:
  return false;
}

return true;
747}

749bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
return isInt<12>(Imm);
751}

753bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
return isInt<12>(Imm);
755}

757// On RV32, 64-bit integers are split into their high and low parts and held
758// in two different registers, so the trunc is free since the low register can
759// just be used.
760bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
  return false;
unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
unsigned DestBits = DstTy->getPrimitiveSizeInBits();
return (SrcBits == 64 && DestBits == 32);
766}

768bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
    !SrcVT.isInteger() || !DstVT.isInteger())
  return false;
unsigned SrcBits = SrcVT.getSizeInBits();
unsigned DestBits = DstVT.getSizeInBits();
return (SrcBits == 64 && DestBits == 32);
775}

777bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
// Zexts are free if they can be combined with a load.
if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
  EVT MemVT = LD->getMemoryVT();
  if ((MemVT == MVT::i8 || MemVT == MVT::i16 ||
       (Subtarget.is64Bit() && MemVT == MVT::i32)) &&
      (LD->getExtensionType() == ISD::NON_EXTLOAD ||
       LD->getExtensionType() == ISD::ZEXTLOAD))
    return true;
}

return TargetLowering::isZExtFree(Val, VT2);
789}

791bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
793}

795bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
return Subtarget.hasStdExtZbb();
797}

799bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
return Subtarget.hasStdExtZbb();
801}

803bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                     bool ForCodeSize) const {
if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
  return false;
if (VT == MVT::f32 && !Subtarget.hasStdExtF())
  return false;
if (VT == MVT::f64 && !Subtarget.hasStdExtD())
  return false;
if (Imm.isNegZero())
  return false;
return Imm.isZero();
814}

816bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
       (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
       (VT == MVT::f64 && Subtarget.hasStdExtD());
820}

822// Changes the condition code and swaps operands if necessary, so the SetCC
823// operation matches one of the comparisons supported directly in the RISC-V
824// ISA.
825static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
switch (CC) {
default:
  break;
case ISD::SETGT:
case ISD::SETLE:
case ISD::SETUGT:
case ISD::SETULE:
  CC = ISD::getSetCCSwappedOperands(CC);
  std::swap(LHS, RHS);
  break;
}
837}

839// Return the RISC-V branch opcode that matches the given DAG integer
840// condition code. The CondCode must be one of those supported by the RISC-V
841// ISA (see normaliseSetCC).
842static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC) {
switch (CC) {
default:
  llvm_unreachable("Unsupported CondCode")::llvm::llvm_unreachable_internal("Unsupported CondCode", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 845);
case ISD::SETEQ:
  return RISCV::BEQ;
case ISD::SETNE:
  return RISCV::BNE;
case ISD::SETLT:
  return RISCV::BLT;
case ISD::SETGE:
  return RISCV::BGE;
case ISD::SETULT:
  return RISCV::BLTU;
case ISD::SETUGE:
  return RISCV::BGEU;
}
859}

861RISCVVLMUL RISCVTargetLowering::getLMUL(MVT VT) {
assert(VT.isScalableVector() && "Expecting a scalable vector type")((VT.isScalableVector() && "Expecting a scalable vector type"
) ? static_cast<void> (0) : __assert_fail ("VT.isScalableVector() && \"Expecting a scalable vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 862, __PRETTY_FUNCTION__));
unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
if (VT.getVectorElementType() == MVT::i1)
  KnownSize *= 8;

switch (KnownSize) {
default:
  llvm_unreachable("Invalid LMUL.")::llvm::llvm_unreachable_internal("Invalid LMUL.", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 869);
case 8:
  return RISCVVLMUL::LMUL_F8;
case 16:
  return RISCVVLMUL::LMUL_F4;
case 32:
  return RISCVVLMUL::LMUL_F2;
case 64:
  return RISCVVLMUL::LMUL_1;
case 128:
  return RISCVVLMUL::LMUL_2;
case 256:
  return RISCVVLMUL::LMUL_4;
case 512:
  return RISCVVLMUL::LMUL_8;
}
885}

887unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVLMUL LMul) {
switch (LMul) {
default:
  llvm_unreachable("Invalid LMUL.")::llvm::llvm_unreachable_internal("Invalid LMUL.", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 890);
case RISCVVLMUL::LMUL_F8:
case RISCVVLMUL::LMUL_F4:
case RISCVVLMUL::LMUL_F2:
case RISCVVLMUL::LMUL_1:
  return RISCV::VRRegClassID;
case RISCVVLMUL::LMUL_2:
  return RISCV::VRM2RegClassID;
case RISCVVLMUL::LMUL_4:
  return RISCV::VRM4RegClassID;
case RISCVVLMUL::LMUL_8:
  return RISCV::VRM8RegClassID;
}
903}

905unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
RISCVVLMUL LMUL = getLMUL(VT);
if (LMUL == RISCVVLMUL::LMUL_F8 || LMUL == RISCVVLMUL::LMUL_F4 ||
    LMUL == RISCVVLMUL::LMUL_F2 || LMUL == RISCVVLMUL::LMUL_1) {
  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
                "Unexpected subreg numbering");
  return RISCV::sub_vrm1_0 + Index;
}
if (LMUL == RISCVVLMUL::LMUL_2) {
  static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
                "Unexpected subreg numbering");
  return RISCV::sub_vrm2_0 + Index;
}
if (LMUL == RISCVVLMUL::LMUL_4) {
  static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
                "Unexpected subreg numbering");
  return RISCV::sub_vrm4_0 + Index;
}
llvm_unreachable("Invalid vector type.")::llvm::llvm_unreachable_internal("Invalid vector type.", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 923);
924}

926unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
if (VT.getVectorElementType() == MVT::i1)
  return RISCV::VRRegClassID;
return getRegClassIDForLMUL(getLMUL(VT));
930}

932// Attempt to decompose a subvector insert/extract between VecVT and
933// SubVecVT via subregister indices. Returns the subregister index that
934// can perform the subvector insert/extract with the given element index, as
935// well as the index corresponding to any leftover subvectors that must be
936// further inserted/extracted within the register class for SubVecVT.
937std::pair<unsigned, unsigned>
938RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
  MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
  const RISCVRegisterInfo *TRI) {
static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
               RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
               RISCV::VRM2RegClassID > RISCV::VRRegClassID),
              "Register classes not ordered");
unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
// Try to compose a subregister index that takes us from the incoming
// LMUL>1 register class down to the outgoing one. At each step we half
// the LMUL:
//   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
// Note that this is not guaranteed to find a subregister index, such as
// when we are extracting from one VR type to another.
unsigned SubRegIdx = RISCV::NoSubRegister;
for (const unsigned RCID :
     {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
  if (VecRegClassID > RCID && SubRegClassID <= RCID) {
    VecVT = VecVT.getHalfNumVectorElementsVT();
    bool IsHi =
        InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
    SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
                                          getSubregIndexByMVT(VecVT, IsHi));
    if (IsHi)
      InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
  }
return {SubRegIdx, InsertExtractIdx};
966}

968// Return the largest legal scalable vector type that matches VT's element type.
969MVT RISCVTargetLowering::getContainerForFixedLengthVector(
  const TargetLowering &TLI, MVT VT, const RISCVSubtarget &Subtarget) {
assert(VT.isFixedLengthVector() && TLI.isTypeLegal(VT) &&((VT.isFixedLengthVector() && TLI.isTypeLegal(VT) &&
 "Expected legal fixed length vector!") ? static_cast<void
> (0) : __assert_fail ("VT.isFixedLengthVector() && TLI.isTypeLegal(VT) && \"Expected legal fixed length vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 972, __PRETTY_FUNCTION__))
       "Expected legal fixed length vector!")((VT.isFixedLengthVector() && TLI.isTypeLegal(VT) &&
 "Expected legal fixed length vector!") ? static_cast<void
> (0) : __assert_fail ("VT.isFixedLengthVector() && TLI.isTypeLegal(VT) && \"Expected legal fixed length vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 972, __PRETTY_FUNCTION__));

unsigned LMul = Subtarget.getLMULForFixedLengthVector(VT);
assert(LMul <= 8 && isPowerOf2_32(LMul) && "Unexpected LMUL!")((LMul <= 8 && isPowerOf2_32(LMul) && "Unexpected LMUL!"
) ? static_cast<void> (0) : __assert_fail ("LMul <= 8 && isPowerOf2_32(LMul) && \"Unexpected LMUL!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 975, __PRETTY_FUNCTION__));

MVT EltVT = VT.getVectorElementType();
switch (EltVT.SimpleTy) {
default:
  llvm_unreachable("unexpected element type for RVV container")::llvm::llvm_unreachable_internal("unexpected element type for RVV container"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 980);
case MVT::i1: {
  // Masks are calculated assuming 8-bit elements since that's when we need
  // the most elements.
  unsigned EltsPerBlock = RISCV::RVVBitsPerBlock / 8;
  return MVT::getScalableVectorVT(MVT::i1, LMul * EltsPerBlock);
}
case MVT::i8:
case MVT::i16:
case MVT::i32:
case MVT::i64:
case MVT::f16:
case MVT::f32:
case MVT::f64: {
  unsigned EltsPerBlock = RISCV::RVVBitsPerBlock / EltVT.getSizeInBits();
  return MVT::getScalableVectorVT(EltVT, LMul * EltsPerBlock);
}
}
998}

1000MVT RISCVTargetLowering::getContainerForFixedLengthVector(
  SelectionDAG &DAG, MVT VT, const RISCVSubtarget &Subtarget) {
return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
                                        Subtarget);
1004}

1006MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
return getContainerForFixedLengthVector(*this, VT, getSubtarget());
1008}

1010// Grow V to consume an entire RVV register.
1011static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
                                     const RISCVSubtarget &Subtarget) {
assert(VT.isScalableVector() &&((VT.isScalableVector() && "Expected to convert into a scalable vector!"
) ? static_cast<void> (0) : __assert_fail ("VT.isScalableVector() && \"Expected to convert into a scalable vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1014, __PRETTY_FUNCTION__))
       "Expected to convert into a scalable vector!")((VT.isScalableVector() && "Expected to convert into a scalable vector!"
) ? static_cast<void> (0) : __assert_fail ("VT.isScalableVector() && \"Expected to convert into a scalable vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1014, __PRETTY_FUNCTION__));
assert(V.getValueType().isFixedLengthVector() &&((V.getValueType().isFixedLengthVector() && "Expected a fixed length vector operand!"
) ? static_cast<void> (0) : __assert_fail ("V.getValueType().isFixedLengthVector() && \"Expected a fixed length vector operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1016, __PRETTY_FUNCTION__))
       "Expected a fixed length vector operand!")((V.getValueType().isFixedLengthVector() && "Expected a fixed length vector operand!"
) ? static_cast<void> (0) : __assert_fail ("V.getValueType().isFixedLengthVector() && \"Expected a fixed length vector operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1016, __PRETTY_FUNCTION__));
SDLoc DL(V);
SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1020}

1022// Shrink V so it's just big enough to maintain a VT's worth of data.
1023static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
                                       const RISCVSubtarget &Subtarget) {
assert(VT.isFixedLengthVector() &&((VT.isFixedLengthVector() && "Expected to convert into a fixed length vector!"
) ? static_cast<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Expected to convert into a fixed length vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1026, __PRETTY_FUNCTION__))
       "Expected to convert into a fixed length vector!")((VT.isFixedLengthVector() && "Expected to convert into a fixed length vector!"
) ? static_cast<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Expected to convert into a fixed length vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1026, __PRETTY_FUNCTION__));
assert(V.getValueType().isScalableVector() &&((V.getValueType().isScalableVector() && "Expected a scalable vector operand!"
) ? static_cast<void> (0) : __assert_fail ("V.getValueType().isScalableVector() && \"Expected a scalable vector operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1028, __PRETTY_FUNCTION__))
       "Expected a scalable vector operand!")((V.getValueType().isScalableVector() && "Expected a scalable vector operand!"
) ? static_cast<void> (0) : __assert_fail ("V.getValueType().isScalableVector() && \"Expected a scalable vector operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1028, __PRETTY_FUNCTION__));
SDLoc DL(V);
SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1032}

1034// Gets the two common "VL" operands: an all-ones mask and the vector length.
1035// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1036// the vector type that it is contained in.
1037static std::pair<SDValue, SDValue>
1038getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
              const RISCVSubtarget &Subtarget) {
assert(ContainerVT.isScalableVector() && "Expecting scalable container type")((ContainerVT.isScalableVector() && "Expecting scalable container type"
) ? static_cast<void> (0) : __assert_fail ("ContainerVT.isScalableVector() && \"Expecting scalable container type\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1040, __PRETTY_FUNCTION__));
MVT XLenVT = Subtarget.getXLenVT();
SDValue VL = VecVT.isFixedLengthVector()
                 ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
                 : DAG.getRegister(RISCV::X0, XLenVT);
MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
return {Mask, VL};
1048}

1050// As above but assuming the given type is a scalable vector type.
1051static std::pair<SDValue, SDValue>
1052getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
                      const RISCVSubtarget &Subtarget) {
assert(VecVT.isScalableVector() && "Expecting a scalable vector")((VecVT.isScalableVector() && "Expecting a scalable vector"
) ? static_cast<void> (0) : __assert_fail ("VecVT.isScalableVector() && \"Expecting a scalable vector\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1054, __PRETTY_FUNCTION__));
return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1056}

1058// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1059// of either is (currently) supported. This can get us into an infinite loop
1060// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1061// as a ..., etc.
1062// Until either (or both) of these can reliably lower any node, reporting that
1063// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1064// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1065// which is not desirable.
1066bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
  EVT VT, unsigned DefinedValues) const {
return false;
1069}

1071static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                               const RISCVSubtarget &Subtarget) {
MVT VT = Op.getSimpleValueType();
assert(VT.isFixedLengthVector() && "Unexpected vector!")((VT.isFixedLengthVector() && "Unexpected vector!") ?
 static_cast<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Unexpected vector!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1074, __PRETTY_FUNCTION__));

MVT ContainerVT =
    RISCVTargetLowering::getContainerForFixedLengthVector(DAG, VT, Subtarget);

SDLoc DL(Op);
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);

if (VT.getVectorElementType() == MVT::i1) {
  if (ISD::isBuildVectorAllZeros(Op.getNode())) {
    SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
    return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
  }

  if (ISD::isBuildVectorAllOnes(Op.getNode())) {
    SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
    return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
  }

  return SDValue();
}

if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
  unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
                                      : RISCVISD::VMV_V_X_VL;
  Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
  return convertFromScalableVector(VT, Splat, DAG, Subtarget);
}

// Try and match an index sequence, which we can lower directly to the vid
// instruction. An all-undef vector is matched by getSplatValue, above.
if (VT.isInteger()) {
  bool IsVID = true;
  for (unsigned i = 0, e = Op.getNumOperands(); i < e && IsVID; i++)
    IsVID &= Op.getOperand(i).isUndef() ||
             (isa<ConstantSDNode>(Op.getOperand(i)) &&
              Op.getConstantOperandVal(i) == i);

  if (IsVID) {
    SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
    return convertFromScalableVector(VT, VID, DAG, Subtarget);
  }
}

return SDValue();
1120}

1122static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
SDValue V1 = Op.getOperand(0);
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());

if (SVN->isSplat()) {
  int Lane = SVN->getSplatIndex();
  if (Lane >= 0) {
    MVT ContainerVT = RISCVTargetLowering::getContainerForFixedLengthVector(
        DAG, VT, Subtarget);

    V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
    assert(Lane < (int)VT.getVectorNumElements() && "Unexpected lane!")((Lane < (int)VT.getVectorNumElements() && "Unexpected lane!"
) ? static_cast<void> (0) : __assert_fail ("Lane < (int)VT.getVectorNumElements() && \"Unexpected lane!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1136, __PRETTY_FUNCTION__));

    SDValue Mask, VL;
    std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
    MVT XLenVT = Subtarget.getXLenVT();
    SDValue Gather =
        DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
                    DAG.getConstant(Lane, DL, XLenVT), Mask, VL);
    return convertFromScalableVector(VT, Gather, DAG, Subtarget);
  }
}

return SDValue();
1149}

1151static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
                                   SDLoc DL, SelectionDAG &DAG,
                                   const RISCVSubtarget &Subtarget) {
if (VT.isScalableVector())
  return DAG.getFPExtendOrRound(Op, DL, VT);
assert(VT.isFixedLengthVector() &&((VT.isFixedLengthVector() && "Unexpected value type for RVV FP extend/round lowering"
) ? static_cast<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Unexpected value type for RVV FP extend/round lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1157, __PRETTY_FUNCTION__))
       "Unexpected value type for RVV FP extend/round lowering")((VT.isFixedLengthVector() && "Unexpected value type for RVV FP extend/round lowering"
) ? static_cast<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Unexpected value type for RVV FP extend/round lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1157, __PRETTY_FUNCTION__));
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
                      ? RISCVISD::FP_EXTEND_VL
                      : RISCVISD::FP_ROUND_VL;
return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
1164}

1166SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                          SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
1
Control jumps to 'case SPLAT_VECTOR:'  at line 1317→
default:
  report_fatal_error("unimplemented operand");
case ISD::GlobalAddress:
  return lowerGlobalAddress(Op, DAG);
case ISD::BlockAddress:
  return lowerBlockAddress(Op, DAG);
case ISD::ConstantPool:
  return lowerConstantPool(Op, DAG);
case ISD::JumpTable:
  return lowerJumpTable(Op, DAG);
case ISD::GlobalTLSAddress:
  return lowerGlobalTLSAddress(Op, DAG);
case ISD::SELECT:
  return lowerSELECT(Op, DAG);
case ISD::VASTART:
  return lowerVASTART(Op, DAG);
case ISD::FRAMEADDR:
  return lowerFRAMEADDR(Op, DAG);
case ISD::RETURNADDR:
  return lowerRETURNADDR(Op, DAG);
case ISD::SHL_PARTS:
  return lowerShiftLeftParts(Op, DAG);
case ISD::SRA_PARTS:
  return lowerShiftRightParts(Op, DAG, true);
case ISD::SRL_PARTS:
  return lowerShiftRightParts(Op, DAG, false);
case ISD::BITCAST: {
  SDValue Op0 = Op.getOperand(0);
  // We can handle fixed length vector bitcasts with a simple replacement
  // in isel.
  if (Op.getValueType().isFixedLengthVector()) {
    if (Op0.getValueType().isFixedLengthVector())
      return Op;
    return SDValue();
  }
  assert(((Subtarget.is64Bit() && Subtarget.hasStdExtF()) ||((((Subtarget.is64Bit() && Subtarget.hasStdExtF()) ||
 Subtarget.hasStdExtZfh()) && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("((Subtarget.is64Bit() && Subtarget.hasStdExtF()) || Subtarget.hasStdExtZfh()) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1206, __PRETTY_FUNCTION__))
          Subtarget.hasStdExtZfh()) &&((((Subtarget.is64Bit() && Subtarget.hasStdExtF()) ||
 Subtarget.hasStdExtZfh()) && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("((Subtarget.is64Bit() && Subtarget.hasStdExtF()) || Subtarget.hasStdExtZfh()) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1206, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((((Subtarget.is64Bit() && Subtarget.hasStdExtF()) ||
 Subtarget.hasStdExtZfh()) && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("((Subtarget.is64Bit() && Subtarget.hasStdExtF()) || Subtarget.hasStdExtZfh()) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1206, __PRETTY_FUNCTION__));
  SDLoc DL(Op);
  if (Op.getValueType() == MVT::f16 && Subtarget.hasStdExtZfh()) {
    if (Op0.getValueType() != MVT::i16)
      return SDValue();
    SDValue NewOp0 =
        DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Op0);
    SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
    return FPConv;
  } else if (Op.getValueType() == MVT::f32 && Subtarget.is64Bit() &&
             Subtarget.hasStdExtF()) {
    if (Op0.getValueType() != MVT::i32)
      return SDValue();
    SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
    SDValue FPConv =
        DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
    return FPConv;
  }
  return SDValue();
}
case ISD::INTRINSIC_WO_CHAIN:
  return LowerINTRINSIC_WO_CHAIN(Op, DAG);
case ISD::INTRINSIC_W_CHAIN:
  return LowerINTRINSIC_W_CHAIN(Op, DAG);
case ISD::BSWAP:
case ISD::BITREVERSE: {
  // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
  assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation")((Subtarget.hasStdExtZbp() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasStdExtZbp() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1233, __PRETTY_FUNCTION__));
  MVT VT = Op.getSimpleValueType();
  SDLoc DL(Op);
  // Start with the maximum immediate value which is the bitwidth - 1.
  unsigned Imm = VT.getSizeInBits() - 1;
  // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
  if (Op.getOpcode() == ISD::BSWAP)
    Imm &= ~0x7U;
  return DAG.getNode(RISCVISD::GREVI, DL, VT, Op.getOperand(0),
                     DAG.getTargetConstant(Imm, DL, Subtarget.getXLenVT()));
}
case ISD::FSHL:
case ISD::FSHR: {
  MVT VT = Op.getSimpleValueType();
  assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization")((VT == Subtarget.getXLenVT() && "Unexpected custom legalization"
) ? static_cast<void> (0) : __assert_fail ("VT == Subtarget.getXLenVT() && \"Unexpected custom legalization\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1247, __PRETTY_FUNCTION__));
  SDLoc DL(Op);
  // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
  // use log(XLen) bits. Mask the shift amount accordingly.
  unsigned ShAmtWidth = Subtarget.getXLen() - 1;
  SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
                              DAG.getConstant(ShAmtWidth, DL, VT));
  unsigned Opc = Op.getOpcode() == ISD::FSHL ? RISCVISD::FSL : RISCVISD::FSR;
  return DAG.getNode(Opc, DL, VT, Op.getOperand(0), Op.getOperand(1), ShAmt);
}
case ISD::TRUNCATE: {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  // Only custom-lower vector truncates
  if (!VT.isVector())
    return Op;

  // Truncates to mask types are handled differently
  if (VT.getVectorElementType() == MVT::i1)
    return lowerVectorMaskTrunc(Op, DAG);

  // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
  // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
  // truncate by one power of two at a time.
  MVT DstEltVT = VT.getVectorElementType();

  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();
  MVT SrcEltVT = SrcVT.getVectorElementType();

  assert(DstEltVT.bitsLT(SrcEltVT) &&((DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT
.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits
()) && "Unexpected vector truncate lowering") ? static_cast
<void> (0) : __assert_fail ("DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits()) && \"Unexpected vector truncate lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1280, __PRETTY_FUNCTION__))
         isPowerOf2_64(DstEltVT.getSizeInBits()) &&((DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT
.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits
()) && "Unexpected vector truncate lowering") ? static_cast
<void> (0) : __assert_fail ("DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits()) && \"Unexpected vector truncate lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1280, __PRETTY_FUNCTION__))
         isPowerOf2_64(SrcEltVT.getSizeInBits()) &&((DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT
.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits
()) && "Unexpected vector truncate lowering") ? static_cast
<void> (0) : __assert_fail ("DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits()) && \"Unexpected vector truncate lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1280, __PRETTY_FUNCTION__))
         "Unexpected vector truncate lowering")((DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT
.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits
()) && "Unexpected vector truncate lowering") ? static_cast
<void> (0) : __assert_fail ("DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) && isPowerOf2_64(SrcEltVT.getSizeInBits()) && \"Unexpected vector truncate lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1280, __PRETTY_FUNCTION__));

  MVT ContainerVT = SrcVT;
  if (SrcVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(SrcVT);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }

  SDValue Result = Src;
  SDValue Mask, VL;
  std::tie(Mask, VL) =
      getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
  LLVMContext &Context = *DAG.getContext();
  const ElementCount Count = ContainerVT.getVectorElementCount();
  do {
    SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
    EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
    Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
                         Mask, VL);
  } while (SrcEltVT != DstEltVT);

  if (SrcVT.isFixedLengthVector())
    Result = convertFromScalableVector(VT, Result, DAG, Subtarget);

  return Result;
}
case ISD::ANY_EXTEND:
case ISD::ZERO_EXTEND:
  if (Op.getOperand(0).getValueType().isVector() &&
      Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
    return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
  return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
case ISD::SIGN_EXTEND:
  if (Op.getOperand(0).getValueType().isVector() &&
      Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
    return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
  return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
case ISD::SPLAT_VECTOR:
  return lowerSPLATVECTOR(Op, DAG);
2
←
Calling 'RISCVTargetLowering::lowerSPLATVECTOR'→
case ISD::INSERT_VECTOR_ELT:
  return lowerINSERT_VECTOR_ELT(Op, DAG);
case ISD::EXTRACT_VECTOR_ELT:
  return lowerEXTRACT_VECTOR_ELT(Op, DAG);
case ISD::VSCALE: {
  MVT VT = Op.getSimpleValueType();
  SDLoc DL(Op);
  SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
  // We define our scalable vector types for lmul=1 to use a 64 bit known
  // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
  // vscale as VLENB / 8.
  SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
                               DAG.getConstant(3, DL, VT));
  return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
}
case ISD::FP_EXTEND: {
  // RVV can only do fp_extend to types double the size as the source. We
  // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
  // via f32.
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();

  // Prepare any fixed-length vector operands.
  MVT ContainerVT = VT;
  if (SrcVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT);
    MVT SrcContainerVT =
        ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }

  if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
      SrcVT.getVectorElementType() != MVT::f16) {
    // For scalable vectors, we only need to close the gap between
    // vXf16->vXf64.
    if (!VT.isFixedLengthVector())
      return Op;
    // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
    Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
    return convertFromScalableVector(VT, Src, DAG, Subtarget);
  }

  MVT InterVT = VT.changeVectorElementType(MVT::f32);
  MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
  SDValue IntermediateExtend = getRVVFPExtendOrRound(
      Src, InterVT, InterContainerVT, DL, DAG, Subtarget);

  SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
                                         DL, DAG, Subtarget);
  if (VT.isFixedLengthVector())
    return convertFromScalableVector(VT, Extend, DAG, Subtarget);
  return Extend;
}
case ISD::FP_ROUND: {
  // RVV can only do fp_round to types half the size as the source. We
  // custom-lower f64->f16 rounds via RVV's round-to-odd float
  // conversion instruction.
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();

  // Prepare any fixed-length vector operands.
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
    ContainerVT =
        SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }

  if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
      SrcVT.getVectorElementType() != MVT::f64) {
    // For scalable vectors, we only need to close the gap between
    // vXf64<->vXf16.
    if (!VT.isFixedLengthVector())
      return Op;
    // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
    Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
    return convertFromScalableVector(VT, Src, DAG, Subtarget);
  }

  SDValue Mask, VL;
  std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);

  MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
  SDValue IntermediateRound =
      DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
  SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
                                        DL, DAG, Subtarget);

  if (VT.isFixedLengthVector())
    return convertFromScalableVector(VT, Round, DAG, Subtarget);
  return Round;
}
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP: {
  // RVV can only do fp<->int conversions to types half/double the size as
  // the source. We custom-lower any conversions that do two hops into
  // sequences.
  MVT VT = Op.getSimpleValueType();
  if (!VT.isVector())
    return Op;
  SDLoc DL(Op);
  SDValue Src = Op.getOperand(0);
  MVT EltVT = VT.getVectorElementType();
  MVT SrcVT = Src.getSimpleValueType();
  MVT SrcEltVT = SrcVT.getVectorElementType();
  unsigned EltSize = EltVT.getSizeInBits();
  unsigned SrcEltSize = SrcEltVT.getSizeInBits();
  assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&((isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize)
 && "Unexpected vector element types") ? static_cast<
void> (0) : __assert_fail ("isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) && \"Unexpected vector element types\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1434, __PRETTY_FUNCTION__))
         "Unexpected vector element types")((isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize)
 && "Unexpected vector element types") ? static_cast<
void> (0) : __assert_fail ("isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) && \"Unexpected vector element types\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1434, __PRETTY_FUNCTION__));

  bool IsInt2FP = SrcEltVT.isInteger();
  // Widening conversions
  if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
    if (IsInt2FP) {
      // Do a regular integer sign/zero extension then convert to float.
      MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
                                    VT.getVectorElementCount());
      unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
                               ? ISD::ZERO_EXTEND
                               : ISD::SIGN_EXTEND;
      SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
      return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
    }
    // FP2Int
    assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering")((SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering"
) ? static_cast<void> (0) : __assert_fail ("SrcEltVT == MVT::f16 && \"Unexpected FP_TO_[US]INT lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1450, __PRETTY_FUNCTION__));
    // Do one doubling fp_extend then complete the operation by converting
    // to int.
    MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
    SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
    return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
  }

  // Narrowing conversions
  if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
    if (IsInt2FP) {
      // One narrowing int_to_fp, then an fp_round.
      assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering")((EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering"
) ? static_cast<void> (0) : __assert_fail ("EltVT == MVT::f16 && \"Unexpected [US]_TO_FP lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1462, __PRETTY_FUNCTION__));
      MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
      SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
      return DAG.getFPExtendOrRound(Int2FP, DL, VT);
    }
    // FP2Int
    // One narrowing fp_to_int, then truncate the integer. If the float isn't
    // representable by the integer, the result is poison.
    MVT IVecVT =
        MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
                         VT.getVectorElementCount());
    SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
    return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
  }

  // Scalable vectors can exit here. Patterns will handle equally-sized
  // conversions halving/doubling ones.
  if (!VT.isFixedLengthVector())
    return Op;

  // For fixed-length vectors we lower to a custom "VL" node.
  unsigned RVVOpc = 0;
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Impossible opcode")::llvm::llvm_unreachable_internal("Impossible opcode", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1486);
  case ISD::FP_TO_SINT:
    RVVOpc = RISCVISD::FP_TO_SINT_VL;
    break;
  case ISD::FP_TO_UINT:
    RVVOpc = RISCVISD::FP_TO_UINT_VL;
    break;
  case ISD::SINT_TO_FP:
    RVVOpc = RISCVISD::SINT_TO_FP_VL;
    break;
  case ISD::UINT_TO_FP:
    RVVOpc = RISCVISD::UINT_TO_FP_VL;
    break;
  }

  MVT ContainerVT, SrcContainerVT;
  // Derive the reference container type from the larger vector type.
  if (SrcEltSize > EltSize) {
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
    ContainerVT =
        SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
  } else {
    ContainerVT = getContainerForFixedLengthVector(VT);
    SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
  }

  SDValue Mask, VL;
  std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);

  Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
  return convertFromScalableVector(VT, Src, DAG, Subtarget);
}
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_UMIN:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
  return lowerVECREDUCE(Op, DAG);
case ISD::VECREDUCE_FADD:
case ISD::VECREDUCE_SEQ_FADD:
  return lowerFPVECREDUCE(Op, DAG);
case ISD::INSERT_SUBVECTOR:
  return lowerINSERT_SUBVECTOR(Op, DAG);
case ISD::EXTRACT_SUBVECTOR:
  return lowerEXTRACT_SUBVECTOR(Op, DAG);
case ISD::VECTOR_REVERSE:
  return lowerVECTOR_REVERSE(Op, DAG);
case ISD::BUILD_VECTOR:
  return lowerBUILD_VECTOR(Op, DAG, Subtarget);
case ISD::VECTOR_SHUFFLE:
  return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
case ISD::CONCAT_VECTORS: {
  // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
  // better than going through the stack, as the default expansion does.
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected CONCAT_VECTORS lowering")((VT.isFixedLengthVector() && "Unexpected CONCAT_VECTORS lowering"
) ? static_cast<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Unexpected CONCAT_VECTORS lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1546, __PRETTY_FUNCTION__));
  unsigned NumOpElts =
      Op.getOperand(0).getSimpleValueType().getVectorNumElements();
  SDValue Vec = DAG.getUNDEF(VT);
  for (const auto &OpIdx : enumerate(Op->ops()))
    Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, OpIdx.value(),
                      DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
  return Vec;
}
case ISD::LOAD:
  return lowerFixedLengthVectorLoadToRVV(Op, DAG);
case ISD::STORE:
  return lowerFixedLengthVectorStoreToRVV(Op, DAG);
case ISD::SETCC:
  return lowerFixedLengthVectorSetccToRVV(Op, DAG);
case ISD::ADD:
  return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
case ISD::SUB:
  return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
case ISD::MUL:
  return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
case ISD::MULHS:
  return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
case ISD::MULHU:
  return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
case ISD::AND:
  return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
                                            RISCVISD::AND_VL);
case ISD::OR:
  return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
                                            RISCVISD::OR_VL);
case ISD::XOR:
  return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
                                            RISCVISD::XOR_VL);
case ISD::SDIV:
  return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
case ISD::SREM:
  return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
case ISD::UDIV:
  return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
case ISD::UREM:
  return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
case ISD::SHL:
  return lowerToScalableOp(Op, DAG, RISCVISD::SHL_VL);
case ISD::SRA:
  return lowerToScalableOp(Op, DAG, RISCVISD::SRA_VL);
case ISD::SRL:
  return lowerToScalableOp(Op, DAG, RISCVISD::SRL_VL);
case ISD::FADD:
  return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
case ISD::FSUB:
  return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
case ISD::FMUL:
  return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
case ISD::FDIV:
  return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
case ISD::FNEG:
  return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
case ISD::FABS:
  return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
case ISD::FSQRT:
  return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
case ISD::FMA:
  return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
case ISD::SMIN:
  return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
case ISD::SMAX:
  return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
case ISD::UMIN:
  return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
case ISD::UMAX:
  return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
case ISD::ABS:
  return lowerABS(Op, DAG);
case ISD::VSELECT:
  return lowerFixedLengthVectorSelectToRVV(Op, DAG);
}
1623}

1625static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
                           SelectionDAG &DAG, unsigned Flags) {
return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
1628}

1630static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
                           SelectionDAG &DAG, unsigned Flags) {
return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
                                 Flags);
1634}

1636static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
                           SelectionDAG &DAG, unsigned Flags) {
return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
                                 N->getOffset(), Flags);
1640}

1642static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
                           SelectionDAG &DAG, unsigned Flags) {
return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
1645}

1647template <class NodeTy>
1648SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
                                   bool IsLocal) const {
SDLoc DL(N);
EVT Ty = getPointerTy(DAG.getDataLayout());

if (isPositionIndependent()) {
  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
  if (IsLocal)
    // Use PC-relative addressing to access the symbol. This generates the
    // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
    // %pcrel_lo(auipc)).
    return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);

  // Use PC-relative addressing to access the GOT for this symbol, then load
  // the address from the GOT. This generates the pattern (PseudoLA sym),
  // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
  return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
}

switch (getTargetMachine().getCodeModel()) {
default:
  report_fatal_error("Unsupported code model for lowering");
case CodeModel::Small: {
  // Generate a sequence for accessing addresses within the first 2 GiB of
  // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
  SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
  SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
  SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
  return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
}
case CodeModel::Medium: {
  // Generate a sequence for accessing addresses within any 2GiB range within
  // the address space. This generates the pattern (PseudoLLA sym), which
  // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
  return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
}
}
1686}

1688SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
                                              SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT Ty = Op.getValueType();
GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
int64_t Offset = N->getOffset();
MVT XLenVT = Subtarget.getXLenVT();

const GlobalValue *GV = N->getGlobal();
bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
SDValue Addr = getAddr(N, DAG, IsLocal);

// In order to maximise the opportunity for common subexpression elimination,
// emit a separate ADD node for the global address offset instead of folding
// it in the global address node. Later peephole optimisations may choose to
// fold it back in when profitable.
if (Offset != 0)
  return DAG.getNode(ISD::ADD, DL, Ty, Addr,
                     DAG.getConstant(Offset, DL, XLenVT));
return Addr;
1708}

1710SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
                                             SelectionDAG &DAG) const {
BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);

return getAddr(N, DAG);
1715}

1717SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
                                             SelectionDAG &DAG) const {
ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);

return getAddr(N, DAG);
1722}

1724SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
                                          SelectionDAG &DAG) const {
JumpTableSDNode *N = cast<JumpTableSDNode>(Op);

return getAddr(N, DAG);
1729}

1731SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
                                            SelectionDAG &DAG,
                                            bool UseGOT) const {
SDLoc DL(N);
EVT Ty = getPointerTy(DAG.getDataLayout());
const GlobalValue *GV = N->getGlobal();
MVT XLenVT = Subtarget.getXLenVT();

if (UseGOT) {
  // Use PC-relative addressing to access the GOT for this TLS symbol, then
  // load the address from the GOT and add the thread pointer. This generates
  // the pattern (PseudoLA_TLS_IE sym), which expands to
  // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
  SDValue Load =
      SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);

  // Add the thread pointer.
  SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
  return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
}

// Generate a sequence for accessing the address relative to the thread
// pointer, with the appropriate adjustment for the thread pointer offset.
// This generates the pattern
// (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
SDValue AddrHi =
    DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
SDValue AddrAdd =
    DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
SDValue AddrLo =
    DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);

SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
SDValue MNAdd = SDValue(
    DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
    0);
return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
1770}

1772SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
                                             SelectionDAG &DAG) const {
SDLoc DL(N);
EVT Ty = getPointerTy(DAG.getDataLayout());
IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
const GlobalValue *GV = N->getGlobal();

// Use a PC-relative addressing mode to access the global dynamic GOT address.
// This generates the pattern (PseudoLA_TLS_GD sym), which expands to
// (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
SDValue Load =
    SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);

// Prepare argument list to generate call.
ArgListTy Args;
ArgListEntry Entry;
Entry.Node = Load;
Entry.Ty = CallTy;
Args.push_back(Entry);

// Setup call to __tls_get_addr.
TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(DL)
    .setChain(DAG.getEntryNode())
    .setLibCallee(CallingConv::C, CallTy,
                  DAG.getExternalSymbol("__tls_get_addr", Ty),
                  std::move(Args));

return LowerCallTo(CLI).first;
1802}

1804SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT Ty = Op.getValueType();
GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
int64_t Offset = N->getOffset();
MVT XLenVT = Subtarget.getXLenVT();

TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());

if (DAG.getMachineFunction().getFunction().getCallingConv() ==
    CallingConv::GHC)
  report_fatal_error("In GHC calling convention TLS is not supported");

SDValue Addr;
switch (Model) {
case TLSModel::LocalExec:
  Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
  break;
case TLSModel::InitialExec:
  Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
  break;
case TLSModel::LocalDynamic:
case TLSModel::GeneralDynamic:
  Addr = getDynamicTLSAddr(N, DAG);
  break;
}

// In order to maximise the opportunity for common subexpression elimination,
// emit a separate ADD node for the global address offset instead of folding
// it in the global address node. Later peephole optimisations may choose to
// fold it back in when profitable.
if (Offset != 0)
  return DAG.getNode(ISD::ADD, DL, Ty, Addr,
                     DAG.getConstant(Offset, DL, XLenVT));
return Addr;
1840}

1842SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
SDValue CondV = Op.getOperand(0);
SDValue TrueV = Op.getOperand(1);
SDValue FalseV = Op.getOperand(2);
SDLoc DL(Op);
MVT XLenVT = Subtarget.getXLenVT();

// If the result type is XLenVT and CondV is the output of a SETCC node
// which also operated on XLenVT inputs, then merge the SETCC node into the
// lowered RISCVISD::SELECT_CC to take advantage of the integer
// compare+branch instructions. i.e.:
// (select (setcc lhs, rhs, cc), truev, falsev)
// -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC &&
    CondV.getOperand(0).getSimpleValueType() == XLenVT) {
  SDValue LHS = CondV.getOperand(0);
  SDValue RHS = CondV.getOperand(1);
  auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
  ISD::CondCode CCVal = CC->get();

  normaliseSetCC(LHS, RHS, CCVal);

  SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
  return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
}

// Otherwise:
// (select condv, truev, falsev)
// -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
SDValue Zero = DAG.getConstant(0, DL, XLenVT);
SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);

SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};

return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
1878}

1880SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();

SDLoc DL(Op);
SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
                               getPointerTy(MF.getDataLayout()));

// vastart just stores the address of the VarArgsFrameIndex slot into the
// memory location argument.
const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
                    MachinePointerInfo(SV));
1893}

1895SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
                                          SelectionDAG &DAG) const {
const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo &MFI = MF.getFrameInfo();
MFI.setFrameAddressIsTaken(true);
Register FrameReg = RI.getFrameRegister(MF);
int XLenInBytes = Subtarget.getXLen() / 8;

EVT VT = Op.getValueType();
SDLoc DL(Op);
SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
while (Depth--) {
  int Offset = -(XLenInBytes * 2);
  SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
                            DAG.getIntPtrConstant(Offset, DL));
  FrameAddr =
      DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
}
return FrameAddr;
1916}

1918SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
                                           SelectionDAG &DAG) const {
const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo &MFI = MF.getFrameInfo();
MFI.setReturnAddressIsTaken(true);
MVT XLenVT = Subtarget.getXLenVT();
int XLenInBytes = Subtarget.getXLen() / 8;

if (verifyReturnAddressArgumentIsConstant(Op, DAG))
  return SDValue();

EVT VT = Op.getValueType();
SDLoc DL(Op);
unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
if (Depth) {
  int Off = -XLenInBytes;
  SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
  SDValue Offset = DAG.getConstant(Off, DL, VT);
  return DAG.getLoad(VT, DL, DAG.getEntryNode(),
                     DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
                     MachinePointerInfo());
}

// Return the value of the return address register, marking it an implicit
// live-in.
Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
1946}

1948SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
                                               SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue Lo = Op.getOperand(0);
SDValue Hi = Op.getOperand(1);
SDValue Shamt = Op.getOperand(2);
EVT VT = Lo.getValueType();

// if Shamt-XLEN < 0: // Shamt < XLEN
//   Lo = Lo << Shamt
//   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
// else:
//   Lo = 0
//   Hi = Lo << (Shamt-XLEN)

SDValue Zero = DAG.getConstant(0, DL, VT);
SDValue One = DAG.getConstant(1, DL, VT);
SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);

SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
SDValue ShiftRightLo =
    DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);

SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);

Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);

SDValue Parts[2] = {Lo, Hi};
return DAG.getMergeValues(Parts, DL);
1985}

1987SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
                                                bool IsSRA) const {
SDLoc DL(Op);
SDValue Lo = Op.getOperand(0);
SDValue Hi = Op.getOperand(1);
SDValue Shamt = Op.getOperand(2);
EVT VT = Lo.getValueType();

// SRA expansion:
//   if Shamt-XLEN < 0: // Shamt < XLEN
//     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
//     Hi = Hi >>s Shamt
//   else:
//     Lo = Hi >>s (Shamt-XLEN);
//     Hi = Hi >>s (XLEN-1)
//
// SRL expansion:
//   if Shamt-XLEN < 0: // Shamt < XLEN
//     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
//     Hi = Hi >>u Shamt
//   else:
//     Lo = Hi >>u (Shamt-XLEN);
//     Hi = 0;

unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;

SDValue Zero = DAG.getConstant(0, DL, VT);
SDValue One = DAG.getConstant(1, DL, VT);
SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);

SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
SDValue ShiftLeftHi =
    DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
SDValue HiFalse =
    IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;

SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);

Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);

SDValue Parts[2] = {Lo, Hi};
return DAG.getMergeValues(Parts, DL);
2037}

2039// Custom-lower a SPLAT_VECTOR where XLEN<SEW, as the SEW element type is
2040// illegal (currently only vXi64 RV32).
2041// FIXME: We could also catch non-constant sign-extended i32 values and lower
2042// them to SPLAT_VECTOR_I64
2043SDValue RISCVTargetLowering::lowerSPLATVECTOR(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT VecVT = Op.getValueType();
assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&((!Subtarget.is64Bit() && VecVT.getVectorElementType(
) == MVT::i64 && "Unexpected SPLAT_VECTOR lowering") ?
 static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 && \"Unexpected SPLAT_VECTOR lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2048, __PRETTY_FUNCTION__))
3
←
Assuming the condition is true→
4
←
'?' condition is true→
       "Unexpected SPLAT_VECTOR lowering")((!Subtarget.is64Bit() && VecVT.getVectorElementType(
) == MVT::i64 && "Unexpected SPLAT_VECTOR lowering") ?
 static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 && \"Unexpected SPLAT_VECTOR lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2048, __PRETTY_FUNCTION__));
SDValue SplatVal = Op.getOperand(0);
5
←
Value assigned to 'SplatVal.Node'→

// If we can prove that the value is a sign-extended 32-bit value, lower this
// as a custom node in order to try and match RVV vector/scalar instructions.
if (auto *CVal = dyn_cast<ConstantSDNode>(SplatVal)) {
6
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'→
21
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'→
22
←
Assuming 'CVal' is null→
23
←
Taking false branch→
  if (isInt<32>(CVal->getSExtValue()))
    return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT,
                       DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32));
}

if (SplatVal.getOpcode() == ISD::SIGN_EXTEND &&
24
←
Calling 'SDValue::getOpcode'→
    SplatVal.getOperand(0).getValueType() == MVT::i32) {
  return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT,
                     SplatVal.getOperand(0));
}

// Else, on RV32 we lower an i64-element SPLAT_VECTOR thus, being careful not
// to accidentally sign-extend the 32-bit halves to the e64 SEW:
// vmv.v.x vX, hi
// vsll.vx vX, vX, /*32*/
// vmv.v.x vY, lo
// vsll.vx vY, vY, /*32*/
// vsrl.vx vY, vY, /*32*/
// vor.vv vX, vX, vY
SDValue One = DAG.getConstant(1, DL, MVT::i32);
SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
SDValue ThirtyTwoV = DAG.getConstant(32, DL, VecVT);
SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, SplatVal, Zero);
SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, SplatVal, One);

Lo = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
Lo = DAG.getNode(ISD::SHL, DL, VecVT, Lo, ThirtyTwoV);
Lo = DAG.getNode(ISD::SRL, DL, VecVT, Lo, ThirtyTwoV);

if (isNullConstant(Hi))
  return Lo;

Hi = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Hi);
Hi = DAG.getNode(ISD::SHL, DL, VecVT, Hi, ThirtyTwoV);

return DAG.getNode(ISD::OR, DL, VecVT, Lo, Hi);
2090}

2092// Custom-lower extensions from mask vectors by using a vselect either with 1
2093// for zero/any-extension or -1 for sign-extension:
2094//   (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
2095// Note that any-extension is lowered identically to zero-extension.
2096SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
                                              int64_t ExtTrueVal) const {
SDLoc DL(Op);
MVT VecVT = Op.getSimpleValueType();
SDValue Src = Op.getOperand(0);
// Only custom-lower extensions from mask types
assert(Src.getValueType().isVector() &&((Src.getValueType().isVector() && Src.getValueType()
.getVectorElementType() == MVT::i1) ? static_cast<void>
 (0) : __assert_fail ("Src.getValueType().isVector() && Src.getValueType().getVectorElementType() == MVT::i1"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2103, __PRETTY_FUNCTION__))
       Src.getValueType().getVectorElementType() == MVT::i1)((Src.getValueType().isVector() && Src.getValueType()
.getVectorElementType() == MVT::i1) ? static_cast<void>
 (0) : __assert_fail ("Src.getValueType().isVector() && Src.getValueType().getVectorElementType() == MVT::i1"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2103, __PRETTY_FUNCTION__));

MVT XLenVT = Subtarget.getXLenVT();
SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);

if (VecVT.isScalableVector()) {
  // Be careful not to introduce illegal scalar types at this stage, and be
  // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
  // illegal and must be expanded. Since we know that the constants are
  // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
  bool IsRV32E64 =
      !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;

  if (!IsRV32E64) {
    SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
    SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
  } else {
    SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
    SplatTrueVal =
        DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
  }

  return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
}

MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
MVT I1ContainerVT =
    MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());

SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);

SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
SplatTrueVal =
    DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
                             SplatTrueVal, SplatZero, VL);

return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
2145}

2147SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
  SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
MVT ExtVT = Op.getSimpleValueType();
// Only custom-lower extensions from fixed-length vector types.
if (!ExtVT.isFixedLengthVector())
  return Op;
MVT VT = Op.getOperand(0).getSimpleValueType();
// Grab the canonical container type for the extended type. Infer the smaller
// type from that to ensure the same number of vector elements, as we know
// the LMUL will be sufficient to hold the smaller type.
MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
// Get the extended container type manually to ensure the same number of
// vector elements between source and dest.
MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
                                   ContainerExtVT.getVectorElementCount());

SDValue Op1 =
    convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);

SDLoc DL(Op);
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);

SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);

return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
2173}

2175// Custom-lower truncations from vectors to mask vectors by using a mask and a
2176// setcc operation:
2177//   (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
2178SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
                                                SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT MaskVT = Op.getValueType();
// Only expect to custom-lower truncations to mask types
assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&((MaskVT.isVector() && MaskVT.getVectorElementType() ==
 MVT::i1 && "Unexpected type for vector mask lowering"
) ? static_cast<void> (0) : __assert_fail ("MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 && \"Unexpected type for vector mask lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2184, __PRETTY_FUNCTION__))
       "Unexpected type for vector mask lowering")((MaskVT.isVector() && MaskVT.getVectorElementType() ==
 MVT::i1 && "Unexpected type for vector mask lowering"
) ? static_cast<void> (0) : __assert_fail ("MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 && \"Unexpected type for vector mask lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2184, __PRETTY_FUNCTION__));
SDValue Src = Op.getOperand(0);
MVT VecVT = Src.getSimpleValueType();

// If this is a fixed vector, we need to convert it to a scalable vector.
MVT ContainerVT = VecVT;
if (VecVT.isFixedLengthVector()) {
  ContainerVT = getContainerForFixedLengthVector(VecVT);
  Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
}

SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());

SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);

if (VecVT.isScalableVector()) {
  SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
  return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
}

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);

MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
SDValue Trunc =
    DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
                    DAG.getCondCode(ISD::SETNE), Mask, VL);
return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
2215}

2217SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
                                                  SelectionDAG &DAG) const {
SDLoc DL(Op);
MVT VecVT = Op.getSimpleValueType();
SDValue Vec = Op.getOperand(0);
SDValue Val = Op.getOperand(1);
SDValue Idx = Op.getOperand(2);

MVT ContainerVT = VecVT;
// If the operand is a fixed-length vector, convert to a scalable one.
if (VecVT.isFixedLengthVector()) {
  ContainerVT = getContainerForFixedLengthVector(VecVT);
  Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
}

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);

// Custom-legalize INSERT_VECTOR_ELT where XLEN>=SEW, so that the vector is
// first slid down into position, the value is inserted into the first
// position, and the vector is slid back up. We do this to simplify patterns.
//   (slideup vec, (insertelt (slidedown impdef, vec, idx), val, 0), idx),
if (Subtarget.is64Bit() || Val.getValueType() != MVT::i64) {
  if (isNullConstant(Idx))
    return DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT, Vec, Val, VL);
  SDValue Slidedown =
      DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
                  DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
  SDValue InsertElt0 =
      DAG.getNode(RISCVISD::VMV_S_XF_VL, DL, ContainerVT, Slidedown, Val, VL);
  return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec, InsertElt0,
                     Idx, Mask, VL);
}

// Custom-legalize INSERT_VECTOR_ELT where XLEN<SEW, as the SEW element type
// is illegal (currently only vXi64 RV32).
// Since there is no easy way of getting a single element into a vector when
// XLEN<SEW, we lower the operation to the following sequence:
//   splat      vVal, rVal
//   vid.v      vVid
//   vmseq.vx   mMask, vVid, rIdx
//   vmerge.vvm vDest, vSrc, vVal, mMask
// This essentially merges the original vector with the inserted element by
// using a mask whose only set bit is that corresponding to the insert
// index.
SDValue SplattedVal = DAG.getSplatVector(ContainerVT, DL, Val);
SDValue SplattedIdx =
    DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, Idx, VL);

SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
auto SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ContainerVT);
SDValue SelectCond =
    DAG.getNode(RISCVISD::SETCC_VL, DL, SetCCVT, VID, SplattedIdx,
                DAG.getCondCode(ISD::SETEQ), Mask, VL);
SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT,
                             SelectCond, SplattedVal, Vec, VL);
if (!VecVT.isFixedLengthVector())
  return Select;
return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
2277}

2279// Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
2280// extract the first element: (extractelt (slidedown vec, idx), 0). For integer
2281// types this is done using VMV_X_S to allow us to glean information about the
2282// sign bits of the result.
2283SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                   SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue Idx = Op.getOperand(1);
SDValue Vec = Op.getOperand(0);
EVT EltVT = Op.getValueType();
MVT VecVT = Vec.getSimpleValueType();
MVT XLenVT = Subtarget.getXLenVT();

// If this is a fixed vector, we need to convert it to a scalable vector.
MVT ContainerVT = VecVT;
if (VecVT.isFixedLengthVector()) {
  ContainerVT = getContainerForFixedLengthVector(VecVT);
  Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
}

// If the index is 0, the vector is already in the right position.
if (!isNullConstant(Idx)) {
  // Use a VL of 1 to avoid processing more elements than we need.
  SDValue VL = DAG.getConstant(1, DL, XLenVT);
  MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
  SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
  Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
                    DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
}

if (!EltVT.isInteger()) {
  // Floating-point extracts are handled in TableGen.
  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
                     DAG.getConstant(0, DL, XLenVT));
}

SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
2317}

2319SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                   SelectionDAG &DAG) const {
unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
SDLoc DL(Op);

if (Subtarget.hasStdExtV()) {
  // Some RVV intrinsics may claim that they want an integer operand to be
  // extended.
  if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
          RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo)) {
    if (II->ExtendedOperand) {
      assert(II->ExtendedOperand < Op.getNumOperands())((II->ExtendedOperand < Op.getNumOperands()) ? static_cast
<void> (0) : __assert_fail ("II->ExtendedOperand < Op.getNumOperands()"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2330, __PRETTY_FUNCTION__));
      SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
      SDValue &ScalarOp = Operands[II->ExtendedOperand];
      EVT OpVT = ScalarOp.getValueType();
      if (OpVT == MVT::i8 || OpVT == MVT::i16 ||
          (OpVT == MVT::i32 && Subtarget.is64Bit())) {
        // If the operand is a constant, sign extend to increase our chances
        // of being able to use a .vi instruction. ANY_EXTEND would become a
        // a zero extend and the simm5 check in isel would fail.
        // FIXME: Should we ignore the upper bits in isel instead?
        unsigned ExtOpc = isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND
                                                        : ISD::ANY_EXTEND;
        ScalarOp = DAG.getNode(ExtOpc, DL, Subtarget.getXLenVT(), ScalarOp);
        return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
                           Operands);
      }
    }
  }
}

switch (IntNo) {
default:
  return SDValue();    // Don't custom lower most intrinsics.
case Intrinsic::thread_pointer: {
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  return DAG.getRegister(RISCV::X4, PtrVT);
}
case Intrinsic::riscv_vmv_x_s:
  assert(Op.getValueType() == Subtarget.getXLenVT() && "Unexpected VT!")((Op.getValueType() == Subtarget.getXLenVT() && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType() == Subtarget.getXLenVT() && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2358, __PRETTY_FUNCTION__));
  return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
                     Op.getOperand(1));
case Intrinsic::riscv_vmv_v_x: {
  SDValue Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(),
                               Op.getOperand(1));
  return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, Op.getValueType(),
                     Scalar, Op.getOperand(2));
}
case Intrinsic::riscv_vfmv_v_f:
  return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
                     Op.getOperand(1), Op.getOperand(2));
}
2371}

2373SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
                                                  SelectionDAG &DAG) const {
unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
SDLoc DL(Op);

if (Subtarget.hasStdExtV()) {
  // Some RVV intrinsics may claim that they want an integer operand to be
  // extended.
  if (const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
          RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo)) {
    if (II->ExtendedOperand) {
      // The operands start from the second argument in INTRINSIC_W_CHAIN.
      unsigned ExtendOp = II->ExtendedOperand + 1;
      assert(ExtendOp < Op.getNumOperands())((ExtendOp < Op.getNumOperands()) ? static_cast<void>
 (0) : __assert_fail ("ExtendOp < Op.getNumOperands()", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2386, __PRETTY_FUNCTION__));
      SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
      SDValue &ScalarOp = Operands[ExtendOp];
      EVT OpVT = ScalarOp.getValueType();
      if (OpVT == MVT::i8 || OpVT == MVT::i16 ||
          (OpVT == MVT::i32 && Subtarget.is64Bit())) {
        // If the operand is a constant, sign extend to increase our chances
        // of being able to use a .vi instruction. ANY_EXTEND would become a
        // a zero extend and the simm5 check in isel would fail.
        // FIXME: Should we ignore the upper bits in isel instead?
        unsigned ExtOpc = isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND
                                                        : ISD::ANY_EXTEND;
        ScalarOp = DAG.getNode(ExtOpc, DL, Subtarget.getXLenVT(), ScalarOp);
        return DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, Op->getVTList(),
                           Operands);
      }
    }
  }
}

return SDValue(); // Don't custom lower most intrinsics.
2407}

2409static MVT getLMUL1VT(MVT VT) {
assert(VT.getVectorElementType().getSizeInBits() <= 64 &&((VT.getVectorElementType().getSizeInBits() <= 64 &&
 "Unexpected vector MVT") ? static_cast<void> (0) : __assert_fail
 ("VT.getVectorElementType().getSizeInBits() <= 64 && \"Unexpected vector MVT\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2411, __PRETTY_FUNCTION__))
       "Unexpected vector MVT")((VT.getVectorElementType().getSizeInBits() <= 64 &&
 "Unexpected vector MVT") ? static_cast<void> (0) : __assert_fail
 ("VT.getVectorElementType().getSizeInBits() <= 64 && \"Unexpected vector MVT\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2411, __PRETTY_FUNCTION__));
return MVT::getScalableVectorVT(
    VT.getVectorElementType(),
    RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
2415}

2417static unsigned getRVVReductionOp(unsigned ISDOpcode) {
switch (ISDOpcode) {
default:
  llvm_unreachable("Unhandled reduction")::llvm::llvm_unreachable_internal("Unhandled reduction", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2420);
case ISD::VECREDUCE_ADD:
  return RISCVISD::VECREDUCE_ADD_VL;
case ISD::VECREDUCE_UMAX:
  return RISCVISD::VECREDUCE_UMAX_VL;
case ISD::VECREDUCE_SMAX:
  return RISCVISD::VECREDUCE_SMAX_VL;
case ISD::VECREDUCE_UMIN:
  return RISCVISD::VECREDUCE_UMIN_VL;
case ISD::VECREDUCE_SMIN:
  return RISCVISD::VECREDUCE_SMIN_VL;
case ISD::VECREDUCE_AND:
  return RISCVISD::VECREDUCE_AND_VL;
case ISD::VECREDUCE_OR:
  return RISCVISD::VECREDUCE_OR_VL;
case ISD::VECREDUCE_XOR:
  return RISCVISD::VECREDUCE_XOR_VL;
}
2438}

2440// Take a (supported) standard ISD reduction opcode and transform it to a RISCV
2441// reduction opcode. Note that this returns a vector type, which must be
2442// further processed to access the scalar result in element 0.
2443SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
                                          SelectionDAG &DAG) const {
SDLoc DL(Op);
MVT VecVT = Op.getOperand(0).getSimpleValueType();
MVT VecEltVT = VecVT.getVectorElementType();

// Avoid creating vectors with illegal type.
if (!isTypeLegal(VecVT))
  return SDValue();

unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());

SDValue Vec = Op.getOperand(0);

MVT ContainerVT = VecVT;
if (VecVT.isFixedLengthVector()) {
  ContainerVT = getContainerForFixedLengthVector(VecVT);
  Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
}

MVT M1VT = getLMUL1VT(ContainerVT);

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);

// FIXME: This is a VLMAX splat which might be too large and can prevent
// vsetvli removal.
SDValue NeutralElem = DAG.getNeutralElement(
    ISD::getVecReduceBaseOpcode(Op.getOpcode()), DL, VecEltVT, SDNodeFlags());
SDValue IdentitySplat = DAG.getSplatVector(M1VT, DL, NeutralElem);
SDValue Reduction =
    DAG.getNode(RVVOpcode, DL, M1VT, Vec, IdentitySplat, Mask, VL);
SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
                           DAG.getConstant(0, DL, Subtarget.getXLenVT()));
return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
2478}

2480// Given a reduction op, this function returns the matching reduction opcode,
2481// the vector SDValue and the scalar SDValue required to lower this to a
2482// RISCVISD node.
2483static std::tuple<unsigned, SDValue, SDValue>
2484getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
SDLoc DL(Op);
switch (Op.getOpcode()) {
default:
  llvm_unreachable("Unhandled reduction")::llvm::llvm_unreachable_internal("Unhandled reduction", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2488);
case ISD::VECREDUCE_FADD:
  return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0),
                         DAG.getConstantFP(0.0, DL, EltVT));
case ISD::VECREDUCE_SEQ_FADD:
  return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
                         Op.getOperand(0));
}
2496}

2498SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
MVT VecEltVT = Op.getSimpleValueType();

unsigned RVVOpcode;
SDValue VectorVal, ScalarVal;
std::tie(RVVOpcode, VectorVal, ScalarVal) =
    getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
MVT VecVT = VectorVal.getSimpleValueType();

MVT ContainerVT = VecVT;
if (VecVT.isFixedLengthVector()) {
  ContainerVT = getContainerForFixedLengthVector(VecVT);
  VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
}

MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);

// FIXME: This is a VLMAX splat which might be too large and can prevent
// vsetvli removal.
SDValue ScalarSplat = DAG.getSplatVector(M1VT, DL, ScalarVal);
SDValue Reduction =
    DAG.getNode(RVVOpcode, DL, M1VT, VectorVal, ScalarSplat, Mask, VL);
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
                   DAG.getConstant(0, DL, Subtarget.getXLenVT()));
2527}

2529SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
                                                 SelectionDAG &DAG) const {
SDValue Vec = Op.getOperand(0);
SDValue SubVec = Op.getOperand(1);
MVT VecVT = Vec.getSimpleValueType();
MVT SubVecVT = SubVec.getSimpleValueType();

SDLoc DL(Op);
MVT XLenVT = Subtarget.getXLenVT();
unsigned OrigIdx = Op.getConstantOperandVal(2);
const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();

// We don't have the ability to slide mask vectors up indexed by their i1
// elements; the smallest we can do is i8. Often we are able to bitcast to
// equivalent i8 vectors. Note that when inserting a fixed-length vector
// into a scalable one, we might not necessarily have enough scalable
// elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
if (SubVecVT.getVectorElementType() == MVT::i1 &&
    (OrigIdx != 0 || !Vec.isUndef())) {
  if (VecVT.getVectorMinNumElements() >= 8 &&
      SubVecVT.getVectorMinNumElements() >= 8) {
    assert(OrigIdx % 8 == 0 && "Invalid index")((OrigIdx % 8 == 0 && "Invalid index") ? static_cast<
void> (0) : __assert_fail ("OrigIdx % 8 == 0 && \"Invalid index\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2550, __PRETTY_FUNCTION__));
    assert(VecVT.getVectorMinNumElements() % 8 == 0 &&((VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT
.getVectorMinNumElements() % 8 == 0 && "Unexpected mask vector lowering"
) ? static_cast<void> (0) : __assert_fail ("VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT.getVectorMinNumElements() % 8 == 0 && \"Unexpected mask vector lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2553, __PRETTY_FUNCTION__))
           SubVecVT.getVectorMinNumElements() % 8 == 0 &&((VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT
.getVectorMinNumElements() % 8 == 0 && "Unexpected mask vector lowering"
) ? static_cast<void> (0) : __assert_fail ("VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT.getVectorMinNumElements() % 8 == 0 && \"Unexpected mask vector lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2553, __PRETTY_FUNCTION__))
           "Unexpected mask vector lowering")((VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT
.getVectorMinNumElements() % 8 == 0 && "Unexpected mask vector lowering"
) ? static_cast<void> (0) : __assert_fail ("VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT.getVectorMinNumElements() % 8 == 0 && \"Unexpected mask vector lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2553, __PRETTY_FUNCTION__));
    OrigIdx /= 8;
    SubVecVT =
        MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
                         SubVecVT.isScalableVector());
    VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
                             VecVT.isScalableVector());
    Vec = DAG.getBitcast(VecVT, Vec);
    SubVec = DAG.getBitcast(SubVecVT, SubVec);
  } else {
    // We can't slide this mask vector up indexed by its i1 elements.
    // This poses a problem when we wish to insert a scalable vector which
    // can't be re-expressed as a larger type. Just choose the slow path and
    // extend to a larger type, then truncate back down.
    MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
    MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
    Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
    SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
    Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
                      Op.getOperand(2));
    SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
    return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
  }
}

// If the subvector vector is a fixed-length type, we cannot use subregister
// manipulation to simplify the codegen; we don't know which register of a
// LMUL group contains the specific subvector as we only know the minimum
// register size. Therefore we must slide the vector group up the full
// amount.
if (SubVecVT.isFixedLengthVector()) {
  if (OrigIdx == 0 && Vec.isUndef())
    return Op;
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
  SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
                       DAG.getUNDEF(ContainerVT), SubVec,
                       DAG.getConstant(0, DL, XLenVT));
  SDValue Mask =
      getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
  // Set the vector length to only the number of elements we care about. Note
  // that for slideup this includes the offset.
  SDValue VL =
      DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
  SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
  SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
                                SubVec, SlideupAmt, Mask, VL);
  if (!VecVT.isFixedLengthVector())
    return Slideup;
  return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
}

unsigned SubRegIdx, RemIdx;
std::tie(SubRegIdx, RemIdx) =
    RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
        VecVT, SubVecVT, OrigIdx, TRI);

RISCVVLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
bool IsSubVecPartReg = SubVecLMUL == RISCVVLMUL::LMUL_F2 ||
                       SubVecLMUL == RISCVVLMUL::LMUL_F4 ||
                       SubVecLMUL == RISCVVLMUL::LMUL_F8;

// 1. If the Idx has been completely eliminated and this subvector's size is
// a vector register or a multiple thereof, or the surrounding elements are
// undef, then this is a subvector insert which naturally aligns to a vector
// register. These can easily be handled using subregister manipulation.
// 2. If the subvector is smaller than a vector register, then the insertion
// must preserve the undisturbed elements of the register. We do this by
// lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
// (which resolves to a subregister copy), performing a VSLIDEUP to place the
// subvector within the vector register, and an INSERT_SUBVECTOR of that
// LMUL=1 type back into the larger vector (resolving to another subregister
// operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
// to avoid allocating a large register group to hold our subvector.
if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
  return Op;

// VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
// OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
// (in our case undisturbed). This means we can set up a subvector insertion
// where OFFSET is the insertion offset, and the VL is the OFFSET plus the
// size of the subvector.
MVT InterSubVT = VecVT;
SDValue AlignedExtract = Vec;
unsigned AlignedIdx = OrigIdx - RemIdx;
if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
  InterSubVT = getLMUL1VT(VecVT);
  // Extract a subvector equal to the nearest full vector register type. This
  // should resolve to a EXTRACT_SUBREG instruction.
  AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
                               DAG.getConstant(AlignedIdx, DL, XLenVT));
}

SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
// For scalable vectors this must be further multiplied by vscale.
SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);

// Construct the vector length corresponding to RemIdx + length(SubVecVT).
VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);

SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
                     DAG.getUNDEF(InterSubVT), SubVec,
                     DAG.getConstant(0, DL, XLenVT));

SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
                              AlignedExtract, SubVec, SlideupAmt, Mask, VL);

// If required, insert this subvector back into the correct vector register.
// This should resolve to an INSERT_SUBREG instruction.
if (VecVT.bitsGT(InterSubVT))
  Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
                        DAG.getConstant(AlignedIdx, DL, XLenVT));

// We might have bitcast from a mask type: cast back to the original type if
// required.
return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
2677}

2679SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
                                                  SelectionDAG &DAG) const {
SDValue Vec = Op.getOperand(0);
MVT SubVecVT = Op.getSimpleValueType();
MVT VecVT = Vec.getSimpleValueType();

SDLoc DL(Op);
MVT XLenVT = Subtarget.getXLenVT();
unsigned OrigIdx = Op.getConstantOperandVal(1);
const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();

// We don't have the ability to slide mask vectors down indexed by their i1
// elements; the smallest we can do is i8. Often we are able to bitcast to
// equivalent i8 vectors. Note that when extracting a fixed-length vector
// from a scalable one, we might not necessarily have enough scalable
// elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
  if (VecVT.getVectorMinNumElements() >= 8 &&
      SubVecVT.getVectorMinNumElements() >= 8) {
    assert(OrigIdx % 8 == 0 && "Invalid index")((OrigIdx % 8 == 0 && "Invalid index") ? static_cast<
void> (0) : __assert_fail ("OrigIdx % 8 == 0 && \"Invalid index\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2698, __PRETTY_FUNCTION__));
    assert(VecVT.getVectorMinNumElements() % 8 == 0 &&((VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT
.getVectorMinNumElements() % 8 == 0 && "Unexpected mask vector lowering"
) ? static_cast<void> (0) : __assert_fail ("VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT.getVectorMinNumElements() % 8 == 0 && \"Unexpected mask vector lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2701, __PRETTY_FUNCTION__))
           SubVecVT.getVectorMinNumElements() % 8 == 0 &&((VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT
.getVectorMinNumElements() % 8 == 0 && "Unexpected mask vector lowering"
) ? static_cast<void> (0) : __assert_fail ("VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT.getVectorMinNumElements() % 8 == 0 && \"Unexpected mask vector lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2701, __PRETTY_FUNCTION__))
           "Unexpected mask vector lowering")((VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT
.getVectorMinNumElements() % 8 == 0 && "Unexpected mask vector lowering"
) ? static_cast<void> (0) : __assert_fail ("VecVT.getVectorMinNumElements() % 8 == 0 && SubVecVT.getVectorMinNumElements() % 8 == 0 && \"Unexpected mask vector lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2701, __PRETTY_FUNCTION__));
    OrigIdx /= 8;
    SubVecVT =
        MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
                         SubVecVT.isScalableVector());
    VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
                             VecVT.isScalableVector());
    Vec = DAG.getBitcast(VecVT, Vec);
  } else {
    // We can't slide this mask vector down, indexed by its i1 elements.
    // This poses a problem when we wish to extract a scalable vector which
    // can't be re-expressed as a larger type. Just choose the slow path and
    // extend to a larger type, then truncate back down.
    // TODO: We could probably improve this when extracting certain fixed
    // from fixed, where we can extract as i8 and shift the correct element
    // right to reach the desired subvector?
    MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
    MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
    Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
    Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
                      Op.getOperand(1));
    SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
    return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
  }
}

// If the subvector vector is a fixed-length type, we cannot use subregister
// manipulation to simplify the codegen; we don't know which register of a
// LMUL group contains the specific subvector as we only know the minimum
// register size. Therefore we must slide the vector group down the full
// amount.
if (SubVecVT.isFixedLengthVector()) {
  // With an index of 0 this is a cast-like subvector, which can be performed
  // with subregister operations.
  if (OrigIdx == 0)
    return Op;
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }
  SDValue Mask =
      getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
  // Set the vector length to only the number of elements we care about. This
  // avoids sliding down elements we're going to discard straight away.
  SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
  SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
  SDValue Slidedown =
      DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
                  DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
  // Now we can use a cast-like subvector extract to get the result.
  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
                     DAG.getConstant(0, DL, XLenVT));
}

unsigned SubRegIdx, RemIdx;
std::tie(SubRegIdx, RemIdx) =
    RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
        VecVT, SubVecVT, OrigIdx, TRI);

// If the Idx has been completely eliminated then this is a subvector extract
// which naturally aligns to a vector register. These can easily be handled
// using subregister manipulation.
if (RemIdx == 0)
  return Op;

// Else we must shift our vector register directly to extract the subvector.
// Do this using VSLIDEDOWN.

// If the vector type is an LMUL-group type, extract a subvector equal to the
// nearest full vector register type. This should resolve to a EXTRACT_SUBREG
// instruction.
MVT InterSubVT = VecVT;
if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
  InterSubVT = getLMUL1VT(VecVT);
  Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
                    DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
}

// Slide this vector register down by the desired number of elements in order
// to place the desired subvector starting at element 0.
SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
// For scalable vectors this must be further multiplied by vscale.
SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
SDValue Slidedown =
    DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
                DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);

// Now the vector is in the right position, extract our final subvector. This
// should resolve to a COPY.
Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
                        DAG.getConstant(0, DL, XLenVT));

// We might have bitcast from a mask type: cast back to the original type if
// required.
return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
2800}

2802// Implement vector_reverse using vrgather.vv with indices determined by
2803// subtracting the id of each element from (VLMAX-1). This will convert
2804// the indices like so:
2805// (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
2806// TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
2807SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
                                               SelectionDAG &DAG) const {
SDLoc DL(Op);
MVT VecVT = Op.getSimpleValueType();
unsigned EltSize = VecVT.getScalarSizeInBits();
unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();

unsigned MaxVLMAX = 0;
unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
if (VectorBitsMax != 0)
  MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;

unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
MVT IntVT = VecVT.changeVectorElementTypeToInteger();

// If this is SEW=8 and VLMAX is unknown or more than 256, we need
// to use vrgatherei16.vv.
// TODO: It's also possible to use vrgatherei16.vv for other types to
// decrease register width for the index calculation.
if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
  // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
  // Reverse each half, then reassemble them in reverse order.
  // NOTE: It's also possible that after splitting that VLMAX no longer
  // requires vrgatherei16.vv.
  if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
    SDValue Lo, Hi;
    std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
    EVT LoVT, HiVT;
    std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
    Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
    Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
    // Reassemble the low and high pieces reversed.
    // FIXME: This is a CONCAT_VECTORS.
    SDValue Res =
        DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
                    DAG.getIntPtrConstant(0, DL));
    return DAG.getNode(
        ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
        DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
  }

  // Just promote the int type to i16 which will double the LMUL.
  IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
  GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
}

MVT XLenVT = Subtarget.getXLenVT();
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);

// Calculate VLMAX-1 for the desired SEW.
unsigned MinElts = VecVT.getVectorMinNumElements();
SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
                            DAG.getConstant(MinElts, DL, XLenVT));
SDValue VLMinus1 =
    DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));

// Splat VLMAX-1 taking care to handle SEW==64 on RV32.
bool IsRV32E64 =
    !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
SDValue SplatVL;
if (!IsRV32E64)
  SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
else
  SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);

SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
SDValue Indices =
    DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);

return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
2878}

2880SDValue
2881RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
                                                   SelectionDAG &DAG) const {
auto *Load = cast<LoadSDNode>(Op);

SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(VT);

SDValue VL =
    DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());

SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
SDValue NewLoad = DAG.getMemIntrinsicNode(
    RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
    Load->getMemoryVT(), Load->getMemOperand());

SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
return DAG.getMergeValues({Result, Load->getChain()}, DL);
2899}

2901SDValue
2902RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
                                                    SelectionDAG &DAG) const {
auto *Store = cast<StoreSDNode>(Op);

SDLoc DL(Op);
MVT VT = Store->getValue().getSimpleValueType();

// FIXME: We probably need to zero any extra bits in a byte for mask stores.
// This is tricky to do.

MVT ContainerVT = getContainerForFixedLengthVector(VT);

SDValue VL =
    DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());

SDValue NewValue =
    convertToScalableVector(ContainerVT, Store->getValue(), DAG, Subtarget);
return DAG.getMemIntrinsicNode(
    RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
    {Store->getChain(), NewValue, Store->getBasePtr(), VL},
    Store->getMemoryVT(), Store->getMemOperand());
2923}

2925SDValue
2926RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
                                                    SelectionDAG &DAG) const {
MVT InVT = Op.getOperand(0).getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(InVT);

MVT VT = Op.getSimpleValueType();

SDValue Op1 =
    convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
SDValue Op2 =
    convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);

SDLoc DL(Op);
SDValue VL =
    DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());

ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();

bool Invert = false;
Optional<unsigned> LogicOpc;
if (ContainerVT.isFloatingPoint()) {
  bool Swap = false;
  switch (CC) {
  default:
    break;
  case ISD::SETULE:
  case ISD::SETULT:
    Swap = true;
    LLVM_FALLTHROUGH[[gnu::fallthrough]];
  case ISD::SETUGE:
  case ISD::SETUGT:
    CC = getSetCCInverse(CC, ContainerVT);
    Invert = true;
    break;
  case ISD::SETOGE:
  case ISD::SETOGT:
  case ISD::SETGE:
  case ISD::SETGT:
    Swap = true;
    break;
  case ISD::SETUEQ:
    // Use !((OLT Op1, Op2) || (OLT Op2, Op1))
    Invert = true;
    LogicOpc = RISCVISD::VMOR_VL;
    CC = ISD::SETOLT;
    break;
  case ISD::SETONE:
    // Use ((OLT Op1, Op2) || (OLT Op2, Op1))
    LogicOpc = RISCVISD::VMOR_VL;
    CC = ISD::SETOLT;
    break;
  case ISD::SETO:
    // Use (OEQ Op1, Op1) && (OEQ Op2, Op2)
    LogicOpc = RISCVISD::VMAND_VL;
    CC = ISD::SETOEQ;
    break;
  case ISD::SETUO:
    // Use (UNE Op1, Op1) || (UNE Op2, Op2)
    LogicOpc = RISCVISD::VMOR_VL;
    CC = ISD::SETUNE;
    break;
  }

  if (Swap) {
    CC = getSetCCSwappedOperands(CC);
    std::swap(Op1, Op2);
  }
}

MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);

// There are 3 cases we need to emit.
// 1. For (OEQ Op1, Op1) && (OEQ Op2, Op2) or (UNE Op1, Op1) || (UNE Op2, Op2)
//    we need to compare each operand with itself.
// 2. For (OLT Op1, Op2) || (OLT Op2, Op1) we need to compare Op1 and Op2 in
//    both orders.
// 3. For any other case we just need one compare with Op1 and Op2.
SDValue Cmp;
if (LogicOpc && (CC == ISD::SETOEQ || CC == ISD::SETUNE)) {
  Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op1,
                    DAG.getCondCode(CC), Mask, VL);
  SDValue Cmp2 = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op2, Op2,
                             DAG.getCondCode(CC), Mask, VL);
  Cmp = DAG.getNode(*LogicOpc, DL, MaskVT, Cmp, Cmp2, VL);
} else {
  Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
                    DAG.getCondCode(CC), Mask, VL);
  if (LogicOpc) {
    SDValue Cmp2 = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op2, Op1,
                               DAG.getCondCode(CC), Mask, VL);
    Cmp = DAG.getNode(*LogicOpc, DL, MaskVT, Cmp, Cmp2, VL);
  }
}

if (Invert) {
  SDValue AllOnes = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
  Cmp = DAG.getNode(RISCVISD::VMXOR_VL, DL, MaskVT, Cmp, AllOnes, VL);
}

return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
3027}

3029SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
  SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
MVT VT = Op.getSimpleValueType();

if (VT.getVectorElementType() == MVT::i1)
  return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);

return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
3037}

3039// Lower vector ABS to smax(X, sub(0, X)).
3040SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
SDLoc DL(Op);
MVT VT = Op.getSimpleValueType();
SDValue X = Op.getOperand(0);

// For scalable vectors we just need to deal with i64 on RV32 since the
// default expansion crashes in getConstant.
if (VT.isScalableVector()) {
  assert(!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64 &&((!Subtarget.is64Bit() && VT.getVectorElementType() ==
 MVT::i64 && "Unexpected custom lowering!") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64 && \"Unexpected custom lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3049, __PRETTY_FUNCTION__))
         "Unexpected custom lowering!")((!Subtarget.is64Bit() && VT.getVectorElementType() ==
 MVT::i64 && "Unexpected custom lowering!") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64 && \"Unexpected custom lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3049, __PRETTY_FUNCTION__));
  SDValue SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT,
                                  DAG.getConstant(0, DL, MVT::i32));
  SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, SplatZero, X);
  return DAG.getNode(ISD::SMAX, DL, VT, X, NegX);
}

assert(VT.isFixedLengthVector() && "Unexpected type")((VT.isFixedLengthVector() && "Unexpected type") ? static_cast
<void> (0) : __assert_fail ("VT.isFixedLengthVector() && \"Unexpected type\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3056, __PRETTY_FUNCTION__));

MVT ContainerVT =
    RISCVTargetLowering::getContainerForFixedLengthVector(DAG, VT, Subtarget);
X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);

SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);

SDValue SplatZero =
    DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                DAG.getConstant(0, DL, Subtarget.getXLenVT()));
SDValue NegX =
    DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
SDValue Max =
    DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);

return convertFromScalableVector(VT, Max, DAG, Subtarget);
3074}

3076SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
  SDValue Op, SelectionDAG &DAG) const {
MVT VT = Op.getSimpleValueType();
MVT ContainerVT = getContainerForFixedLengthVector(VT);

MVT I1ContainerVT =
    MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());

SDValue CC =
    convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
SDValue Op1 =
    convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
SDValue Op2 =
    convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);

SDLoc DL(Op);
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);

SDValue Select =
    DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);

return convertFromScalableVector(VT, Select, DAG, Subtarget);
3099}

3101SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
                                             unsigned NewOpc,
                                             bool HasMask) const {
MVT VT = Op.getSimpleValueType();
assert(useRVVForFixedLengthVectorVT(VT) &&((useRVVForFixedLengthVectorVT(VT) && "Only expected to lower fixed length vector operation!"
) ? static_cast<void> (0) : __assert_fail ("useRVVForFixedLengthVectorVT(VT) && \"Only expected to lower fixed length vector operation!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3106, __PRETTY_FUNCTION__))
       "Only expected to lower fixed length vector operation!")((useRVVForFixedLengthVectorVT(VT) && "Only expected to lower fixed length vector operation!"
) ? static_cast<void> (0) : __assert_fail ("useRVVForFixedLengthVectorVT(VT) && \"Only expected to lower fixed length vector operation!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3106, __PRETTY_FUNCTION__));
MVT ContainerVT = getContainerForFixedLengthVector(VT);

// Create list of operands by converting existing ones to scalable types.
SmallVector<SDValue, 6> Ops;
for (const SDValue &V : Op->op_values()) {
  assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!")((!isa<VTSDNode>(V) && "Unexpected VTSDNode node!"
) ? static_cast<void> (0) : __assert_fail ("!isa<VTSDNode>(V) && \"Unexpected VTSDNode node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3112, __PRETTY_FUNCTION__));

  // Pass through non-vector operands.
  if (!V.getValueType().isVector()) {
    Ops.push_back(V);
    continue;
  }

  // "cast" fixed length vector to a scalable vector.
  assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&((useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
 "Only fixed length vectors are supported!") ? static_cast<
void> (0) : __assert_fail ("useRVVForFixedLengthVectorVT(V.getSimpleValueType()) && \"Only fixed length vectors are supported!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3122, __PRETTY_FUNCTION__))
         "Only fixed length vectors are supported!")((useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
 "Only fixed length vectors are supported!") ? static_cast<
void> (0) : __assert_fail ("useRVVForFixedLengthVectorVT(V.getSimpleValueType()) && \"Only fixed length vectors are supported!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3122, __PRETTY_FUNCTION__));
  Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
}

SDLoc DL(Op);
SDValue Mask, VL;
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
if (HasMask)
  Ops.push_back(Mask);
Ops.push_back(VL);

SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
3135}

3137// Returns the opcode of the target-specific SDNode that implements the 32-bit
3138// form of the given Opcode.
3139static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
switch (Opcode) {
default:
  llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3142);
case ISD::SHL:
  return RISCVISD::SLLW;
case ISD::SRA:
  return RISCVISD::SRAW;
case ISD::SRL:
  return RISCVISD::SRLW;
case ISD::SDIV:
  return RISCVISD::DIVW;
case ISD::UDIV:
  return RISCVISD::DIVUW;
case ISD::UREM:
  return RISCVISD::REMUW;
case ISD::ROTL:
  return RISCVISD::ROLW;
case ISD::ROTR:
  return RISCVISD::RORW;
case RISCVISD::GREVI:
  return RISCVISD::GREVIW;
case RISCVISD::GORCI:
  return RISCVISD::GORCIW;
}
3164}

3166// Converts the given 32-bit operation to a target-specific SelectionDAG node.
3167// Because i32 isn't a legal type for RV64, these operations would otherwise
3168// be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
3169// later one because the fact the operation was originally of type i32 is
3170// lost.
3171static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
                                 unsigned ExtOpc = ISD::ANY_EXTEND) {
SDLoc DL(N);
RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
// ReplaceNodeResults requires we maintain the same type for the return value.
return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
3180}

3182// Converts the given 32-bit operation to a i64 operation with signed extension
3183// semantic to reduce the signed extension instructions.
3184static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
SDLoc DL(N);
SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
                             DAG.getValueType(MVT::i32));
return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
3192}

3194void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
                                           SmallVectorImpl<SDValue> &Results,
                                           SelectionDAG &DAG) const {
SDLoc DL(N);
switch (N->getOpcode()) {
default:
  llvm_unreachable("Don't know how to custom type legalize this operation!")::llvm::llvm_unreachable_internal("Don't know how to custom type legalize this operation!"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3200);
case ISD::STRICT_FP_TO_SINT:
case ISD::STRICT_FP_TO_UINT:
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT: {
  bool IsStrict = N->isStrictFPOpcode();
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3207, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3207, __PRETTY_FUNCTION__));
  SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
  // If the FP type needs to be softened, emit a library call using the 'si'
  // version. If we left it to default legalization we'd end up with 'di'. If
  // the FP type doesn't need to be softened just let generic type
  // legalization promote the result type.
  if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
      TargetLowering::TypeSoftenFloat)
    return;
  RTLIB::Libcall LC;
  if (N->getOpcode() == ISD::FP_TO_SINT ||
      N->getOpcode() == ISD::STRICT_FP_TO_SINT)
    LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
  else
    LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
  MakeLibCallOptions CallOptions;
  EVT OpVT = Op0.getValueType();
  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
  SDValue Result;
  std::tie(Result, Chain) =
      makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
  Results.push_back(Result);
  if (IsStrict)
    Results.push_back(Chain);
  break;
}
case ISD::READCYCLECOUNTER: {
  assert(!Subtarget.is64Bit() &&((!Subtarget.is64Bit() && "READCYCLECOUNTER only has custom type legalization on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"READCYCLECOUNTER only has custom type legalization on riscv32\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3236, __PRETTY_FUNCTION__))
         "READCYCLECOUNTER only has custom type legalization on riscv32")((!Subtarget.is64Bit() && "READCYCLECOUNTER only has custom type legalization on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"READCYCLECOUNTER only has custom type legalization on riscv32\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3236, __PRETTY_FUNCTION__));

  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
  SDValue RCW =
      DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));

  Results.push_back(
      DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
  Results.push_back(RCW.getValue(2));
  break;
}
case ISD::ADD:
case ISD::SUB:
case ISD::MUL:
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3251, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3251, __PRETTY_FUNCTION__));
  if (N->getOperand(1).getOpcode() == ISD::Constant)
    return;
  Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
  break;
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3260, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3260, __PRETTY_FUNCTION__));
  if (N->getOperand(1).getOpcode() == ISD::Constant)
    return;
  Results.push_back(customLegalizeToWOp(N, DAG));
  break;
case ISD::ROTL:
case ISD::ROTR:
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3268, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3268, __PRETTY_FUNCTION__));
  Results.push_back(customLegalizeToWOp(N, DAG));
  break;
case ISD::SDIV:
case ISD::UDIV:
case ISD::UREM: {
  MVT VT = N->getSimpleValueType(0);
  assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&(((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("(VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) && Subtarget.is64Bit() && Subtarget.hasStdExtM() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3277, __PRETTY_FUNCTION__))
         Subtarget.is64Bit() && Subtarget.hasStdExtM() &&(((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("(VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) && Subtarget.is64Bit() && Subtarget.hasStdExtM() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3277, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")(((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("(VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) && Subtarget.is64Bit() && Subtarget.hasStdExtM() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3277, __PRETTY_FUNCTION__));
  if (N->getOperand(0).getOpcode() == ISD::Constant ||
      N->getOperand(1).getOpcode() == ISD::Constant)
    return;

  // If the input is i32, use ANY_EXTEND since the W instructions don't read
  // the upper 32 bits. For other types we need to sign or zero extend
  // based on the opcode.
  unsigned ExtOpc = ISD::ANY_EXTEND;
  if (VT != MVT::i32)
    ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
                                         : ISD::ZERO_EXTEND;

  Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
  break;
}
case ISD::BITCAST: {
  assert(((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtF()) || (N->getValueType(
0) == MVT::i16 && Subtarget.hasStdExtZfh())) &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF()) || (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh())) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3297, __PRETTY_FUNCTION__))
           Subtarget.hasStdExtF()) ||((((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtF()) || (N->getValueType(
0) == MVT::i16 && Subtarget.hasStdExtZfh())) &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF()) || (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh())) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3297, __PRETTY_FUNCTION__))
          (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh())) &&((((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtF()) || (N->getValueType(
0) == MVT::i16 && Subtarget.hasStdExtZfh())) &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF()) || (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh())) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3297, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtF()) || (N->getValueType(
0) == MVT::i16 && Subtarget.hasStdExtZfh())) &&
 "Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF()) || (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh())) && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3297, __PRETTY_FUNCTION__));
  SDValue Op0 = N->getOperand(0);
  if (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh()) {
    if (Op0.getValueType() != MVT::f16)
      return;
    SDValue FPConv =
        DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Op0);
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
  } else if (N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
             Subtarget.hasStdExtF()) {
    if (Op0.getValueType() != MVT::f32)
      return;
    SDValue FPConv =
        DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
  }
  break;
}
case RISCVISD::GREVI:
case RISCVISD::GORCI: {
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3318, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3318, __PRETTY_FUNCTION__));
  // This is similar to customLegalizeToWOp, except that we pass the second
  // operand (a TargetConstant) straight through: it is already of type
  // XLenVT.
  SDLoc DL(N);
  RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
  SDValue NewOp0 =
      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
  SDValue NewRes =
      DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, N->getOperand(1));
  // ReplaceNodeResults requires we maintain the same type for the return
  // value.
  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
  break;
}
case RISCVISD::SHFLI: {
  // There is no SHFLIW instruction, but we can just promote the operation.
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3336, __PRETTY_FUNCTION__))
         "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3336, __PRETTY_FUNCTION__));
  SDLoc DL(N);
  SDValue NewOp0 =
      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
  SDValue NewRes =
      DAG.getNode(RISCVISD::SHFLI, DL, MVT::i64, NewOp0, N->getOperand(1));
  // ReplaceNodeResults requires we maintain the same type for the return
  // value.
  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
  break;
}
case ISD::BSWAP:
case ISD::BITREVERSE: {
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtZbp() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbp() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3350, __PRETTY_FUNCTION__))
         Subtarget.hasStdExtZbp() && "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtZbp() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbp() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3350, __PRETTY_FUNCTION__));
  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,
                               N->getOperand(0));
  unsigned Imm = N->getOpcode() == ISD::BITREVERSE ? 31 : 24;
  SDValue GREVIW = DAG.getNode(RISCVISD::GREVIW, DL, MVT::i64, NewOp0,
                               DAG.getTargetConstant(Imm, DL,
                                                     Subtarget.getXLenVT()));
  // ReplaceNodeResults requires we maintain the same type for the return
  // value.
  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, GREVIW));
  break;
}
case ISD::FSHL:
case ISD::FSHR: {
  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtZbt() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbt() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3365, __PRETTY_FUNCTION__))
         Subtarget.hasStdExtZbt() && "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtZbt() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtZbt() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3365, __PRETTY_FUNCTION__));
  SDValue NewOp0 =
      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
  SDValue NewOp1 =
      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
  SDValue NewOp2 =
      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
  // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
  // Mask the shift amount to 5 bits.
  NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
                       DAG.getConstant(0x1f, DL, MVT::i64));
  unsigned Opc =
      N->getOpcode() == ISD::FSHL ? RISCVISD::FSLW : RISCVISD::FSRW;
  SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewOp2);
  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
  break;
}
case ISD::EXTRACT_VECTOR_ELT: {
  // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
  // type is illegal (currently only vXi64 RV32).
  // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
  // transferred to the destination register. We issue two of these from the
  // upper- and lower- halves of the SEW-bit vector element, slid down to the
  // first element.
  SDLoc DL(N);
  SDValue Vec = N->getOperand(0);
  SDValue Idx = N->getOperand(1);

  // The vector type hasn't been legalized yet so we can't issue target
  // specific nodes if it needs legalization.
  // FIXME: We would manually legalize if it's important.
  if (!isTypeLegal(Vec.getValueType()))
    return;

  MVT VecVT = Vec.getSimpleValueType();

  assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&((!Subtarget.is64Bit() && N->getValueType(0) == MVT
::i64 && VecVT.getVectorElementType() == MVT::i64 &&
 "Unexpected EXTRACT_VECTOR_ELT legalization") ? static_cast<
void> (0) : __assert_fail ("!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 && VecVT.getVectorElementType() == MVT::i64 && \"Unexpected EXTRACT_VECTOR_ELT legalization\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3403, __PRETTY_FUNCTION__))
         VecVT.getVectorElementType() == MVT::i64 &&((!Subtarget.is64Bit() && N->getValueType(0) == MVT
::i64 && VecVT.getVectorElementType() == MVT::i64 &&
 "Unexpected EXTRACT_VECTOR_ELT legalization") ? static_cast<
void> (0) : __assert_fail ("!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 && VecVT.getVectorElementType() == MVT::i64 && \"Unexpected EXTRACT_VECTOR_ELT legalization\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3403, __PRETTY_FUNCTION__))
         "Unexpected EXTRACT_VECTOR_ELT legalization")((!Subtarget.is64Bit() && N->getValueType(0) == MVT
::i64 && VecVT.getVectorElementType() == MVT::i64 &&
 "Unexpected EXTRACT_VECTOR_ELT legalization") ? static_cast<
void> (0) : __assert_fail ("!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 && VecVT.getVectorElementType() == MVT::i64 && \"Unexpected EXTRACT_VECTOR_ELT legalization\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3403, __PRETTY_FUNCTION__));

  // If this is a fixed vector, we need to convert it to a scalable vector.
  MVT ContainerVT = VecVT;
  if (VecVT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VecVT);
    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
  }

  MVT XLenVT = Subtarget.getXLenVT();

  // Use a VL of 1 to avoid processing more elements than we need.
  MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
  SDValue VL = DAG.getConstant(1, DL, XLenVT);
  SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);

  // Unless the index is known to be 0, we must slide the vector down to get
  // the desired element into index 0.
  if (!isNullConstant(Idx)) {
    Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
                      DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
  }

  // Extract the lower XLEN bits of the correct vector element.
  SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);

  // To extract the upper XLEN bits of the vector element, shift the first
  // element right by 32 bits and re-extract the lower XLEN bits.
  SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                   DAG.getConstant(32, DL, XLenVT), VL);
  SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
                               ThirtyTwoV, Mask, VL);

  SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);

  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
  break;
}
case ISD::INTRINSIC_WO_CHAIN: {
  unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
  switch (IntNo) {
  default:
    llvm_unreachable(::llvm::llvm_unreachable_internal("Don't know how to custom type legalize this intrinsic!"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3446)
        "Don't know how to custom type legalize this intrinsic!")::llvm::llvm_unreachable_internal("Don't know how to custom type legalize this intrinsic!"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3446);
  case Intrinsic::riscv_vmv_x_s: {
    EVT VT = N->getValueType(0);
    assert((VT == MVT::i8 || VT == MVT::i16 ||(((VT == MVT::i8 || VT == MVT::i16 || (Subtarget.is64Bit() &&
 VT == MVT::i32)) && "Unexpected custom legalisation!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i8 || VT == MVT::i16 || (Subtarget.is64Bit() && VT == MVT::i32)) && \"Unexpected custom legalisation!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3451, __PRETTY_FUNCTION__))
            (Subtarget.is64Bit() && VT == MVT::i32)) &&(((VT == MVT::i8 || VT == MVT::i16 || (Subtarget.is64Bit() &&
 VT == MVT::i32)) && "Unexpected custom legalisation!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i8 || VT == MVT::i16 || (Subtarget.is64Bit() && VT == MVT::i32)) && \"Unexpected custom legalisation!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3451, __PRETTY_FUNCTION__))
           "Unexpected custom legalisation!")(((VT == MVT::i8 || VT == MVT::i16 || (Subtarget.is64Bit() &&
 VT == MVT::i32)) && "Unexpected custom legalisation!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i8 || VT == MVT::i16 || (Subtarget.is64Bit() && VT == MVT::i32)) && \"Unexpected custom legalisation!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3451, __PRETTY_FUNCTION__));
    SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
                                  Subtarget.getXLenVT(), N->getOperand(1));
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
    break;
  }
  }
  break;
}
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMIN:
  // The custom-lowering for these nodes returns a vector whose first element
  // is the result of the reduction. Extract its first element and let the
  // legalization for EXTRACT_VECTOR_ELT do the rest of the job.
  if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
    Results.push_back(V);
  break;
}
3475}

3477// A structure to hold one of the bit-manipulation patterns below. Together, a
3478// SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
3479//   (or (and (shl x, 1), 0xAAAAAAAA),
3480//       (and (srl x, 1), 0x55555555))
3481struct RISCVBitmanipPat {
SDValue Op;
unsigned ShAmt;
bool IsSHL;

bool formsPairWith(const RISCVBitmanipPat &Other) const {
  return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
}
3489};

3491// Matches patterns of the form
3492//   (and (shl x, C2), (C1 << C2))
3493//   (and (srl x, C2), C1)
3494//   (shl (and x, C1), C2)
3495//   (srl (and x, (C1 << C2)), C2)
3496// Where C2 is a power of 2 and C1 has at least that many leading zeroes.
3497// The expected masks for each shift amount are specified in BitmanipMasks where
3498// BitmanipMasks[log2(C2)] specifies the expected C1 value.
3499// The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
3500// BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
3501// XLen is 64.
3502static Optional<RISCVBitmanipPat>
3503matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&(((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
 "Unexpected number of masks") ? static_cast<void> (0) :
 __assert_fail ("(BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) && \"Unexpected number of masks\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3505, __PRETTY_FUNCTION__))
       "Unexpected number of masks")(((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
 "Unexpected number of masks") ? static_cast<void> (0) :
 __assert_fail ("(BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) && \"Unexpected number of masks\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3505, __PRETTY_FUNCTION__));
Optional<uint64_t> Mask;
// Optionally consume a mask around the shift operation.
if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
  Mask = Op.getConstantOperandVal(1);
  Op = Op.getOperand(0);
}
if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
  return None;
bool IsSHL = Op.getOpcode() == ISD::SHL;

if (!isa<ConstantSDNode>(Op.getOperand(1)))
  return None;
uint64_t ShAmt = Op.getConstantOperandVal(1);

unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
if (ShAmt >= Width && !isPowerOf2_64(ShAmt))
  return None;
// If we don't have enough masks for 64 bit, then we must be trying to
// match SHFL so we're only allowed to shift 1/4 of the width.
if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
  return None;

SDValue Src = Op.getOperand(0);

// The expected mask is shifted left when the AND is found around SHL
// patterns.
//   ((x >> 1) & 0x55555555)
//   ((x << 1) & 0xAAAAAAAA)
bool SHLExpMask = IsSHL;

if (!Mask) {
  // Sometimes LLVM keeps the mask as an operand of the shift, typically when
  // the mask is all ones: consume that now.
  if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
    Mask = Src.getConstantOperandVal(1);
    Src = Src.getOperand(0);
    // The expected mask is now in fact shifted left for SRL, so reverse the
    // decision.
    //   ((x & 0xAAAAAAAA) >> 1)
    //   ((x & 0x55555555) << 1)
    SHLExpMask = !SHLExpMask;
  } else {
    // Use a default shifted mask of all-ones if there's no AND, truncated
    // down to the expected width. This simplifies the logic later on.
    Mask = maskTrailingOnes<uint64_t>(Width);
    *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
  }
}

unsigned MaskIdx = Log2_32(ShAmt);
uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);

if (SHLExpMask)
  ExpMask <<= ShAmt;

if (Mask != ExpMask)
  return None;

return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
3565}

3567// Matches any of the following bit-manipulation patterns:
3568//   (and (shl x, 1), (0x55555555 << 1))
3569//   (and (srl x, 1), 0x55555555)
3570//   (shl (and x, 0x55555555), 1)
3571//   (srl (and x, (0x55555555 << 1)), 1)
3572// where the shift amount and mask may vary thus:
3573//   [1]  = 0x55555555 / 0xAAAAAAAA
3574//   [2]  = 0x33333333 / 0xCCCCCCCC
3575//   [4]  = 0x0F0F0F0F / 0xF0F0F0F0
3576//   [8]  = 0x00FF00FF / 0xFF00FF00
3577//   [16] = 0x0000FFFF / 0xFFFFFFFF
3578//   [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
3579static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
// These are the unshifted masks which we use to match bit-manipulation
// patterns. They may be shifted left in certain circumstances.
static const uint64_t BitmanipMasks[] = {
    0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
    0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};

return matchRISCVBitmanipPat(Op, BitmanipMasks);
3587}

3589// Match the following pattern as a GREVI(W) operation
3590//   (or (BITMANIP_SHL x), (BITMANIP_SRL x))
3591static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget) {
assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson")((Subtarget.hasStdExtZbp() && "Expected Zbp extenson"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasStdExtZbp() && \"Expected Zbp extenson\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3593, __PRETTY_FUNCTION__));
EVT VT = Op.getValueType();

if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
  auto LHS = matchGREVIPat(Op.getOperand(0));
  auto RHS = matchGREVIPat(Op.getOperand(1));
  if (LHS && RHS && LHS->formsPairWith(*RHS)) {
    SDLoc DL(Op);
    return DAG.getNode(
        RISCVISD::GREVI, DL, VT, LHS->Op,
        DAG.getTargetConstant(LHS->ShAmt, DL, Subtarget.getXLenVT()));
  }
}
return SDValue();
3607}

3609// Matches any the following pattern as a GORCI(W) operation
3610// 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
3611// 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
3612// 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
3613// Note that with the variant of 3.,
3614//     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
3615// the inner pattern will first be matched as GREVI and then the outer
3616// pattern will be matched to GORC via the first rule above.
3617// 4.  (or (rotl/rotr x, bitwidth/2), x)
3618static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget) {
assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson")((Subtarget.hasStdExtZbp() && "Expected Zbp extenson"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasStdExtZbp() && \"Expected Zbp extenson\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3620, __PRETTY_FUNCTION__));
EVT VT = Op.getValueType();

if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
  SDLoc DL(Op);
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
    if (Reverse.getOpcode() == RISCVISD::GREVI && Reverse.getOperand(0) == X &&
        isPowerOf2_32(Reverse.getConstantOperandVal(1)))
      return DAG.getNode(RISCVISD::GORCI, DL, VT, X, Reverse.getOperand(1));
    // We can also form GORCI from ROTL/ROTR by half the bitwidth.
    if ((Reverse.getOpcode() == ISD::ROTL ||
         Reverse.getOpcode() == ISD::ROTR) &&
        Reverse.getOperand(0) == X &&
        isa<ConstantSDNode>(Reverse.getOperand(1))) {
      uint64_t RotAmt = Reverse.getConstantOperandVal(1);
      if (RotAmt == (VT.getSizeInBits() / 2))
        return DAG.getNode(
            RISCVISD::GORCI, DL, VT, X,
            DAG.getTargetConstant(RotAmt, DL, Subtarget.getXLenVT()));
    }
    return SDValue();
  };

  // Check for either commutable permutation of (or (GREVI x, shamt), x)
  if (SDValue V = MatchOROfReverse(Op0, Op1))
    return V;
  if (SDValue V = MatchOROfReverse(Op1, Op0))
    return V;

  // OR is commutable so canonicalize its OR operand to the left
  if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
    std::swap(Op0, Op1);
  if (Op0.getOpcode() != ISD::OR)
    return SDValue();
  SDValue OrOp0 = Op0.getOperand(0);
  SDValue OrOp1 = Op0.getOperand(1);
  auto LHS = matchGREVIPat(OrOp0);
  // OR is commutable so swap the operands and try again: x might have been
  // on the left
  if (!LHS) {
    std::swap(OrOp0, OrOp1);
    LHS = matchGREVIPat(OrOp0);
  }
  auto RHS = matchGREVIPat(Op1);
  if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
    return DAG.getNode(
        RISCVISD::GORCI, DL, VT, LHS->Op,
        DAG.getTargetConstant(LHS->ShAmt, DL, Subtarget.getXLenVT()));
  }
}
return SDValue();
3674}

3676// Matches any of the following bit-manipulation patterns:
3677//   (and (shl x, 1), (0x22222222 << 1))
3678//   (and (srl x, 1), 0x22222222)
3679//   (shl (and x, 0x22222222), 1)
3680//   (srl (and x, (0x22222222 << 1)), 1)
3681// where the shift amount and mask may vary thus:
3682//   [1]  = 0x22222222 / 0x44444444
3683//   [2]  = 0x0C0C0C0C / 0x3C3C3C3C
3684//   [4]  = 0x00F000F0 / 0x0F000F00
3685//   [8]  = 0x0000FF00 / 0x00FF0000
3686//   [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
3687static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
// These are the unshifted masks which we use to match bit-manipulation
// patterns. They may be shifted left in certain circumstances.
static const uint64_t BitmanipMasks[] = {
    0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
    0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};

return matchRISCVBitmanipPat(Op, BitmanipMasks);
3695}

3697// Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
3698static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget) {
assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson")((Subtarget.hasStdExtZbp() && "Expected Zbp extenson"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasStdExtZbp() && \"Expected Zbp extenson\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3700, __PRETTY_FUNCTION__));
EVT VT = Op.getValueType();

if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
  return SDValue();

SDValue Op0 = Op.getOperand(0);
SDValue Op1 = Op.getOperand(1);

// Or is commutable so canonicalize the second OR to the LHS.
if (Op0.getOpcode() != ISD::OR)
  std::swap(Op0, Op1);
if (Op0.getOpcode() != ISD::OR)
  return SDValue();

// We found an inner OR, so our operands are the operands of the inner OR
// and the other operand of the outer OR.
SDValue A = Op0.getOperand(0);
SDValue B = Op0.getOperand(1);
SDValue C = Op1;

auto Match1 = matchSHFLPat(A);
auto Match2 = matchSHFLPat(B);

// If neither matched, we failed.
if (!Match1 && !Match2)
  return SDValue();

// We had at least one match. if one failed, try the remaining C operand.
if (!Match1) {
  std::swap(A, C);
  Match1 = matchSHFLPat(A);
  if (!Match1)
    return SDValue();
} else if (!Match2) {
  std::swap(B, C);
  Match2 = matchSHFLPat(B);
  if (!Match2)
    return SDValue();
}
assert(Match1 && Match2)((Match1 && Match2) ? static_cast<void> (0) : __assert_fail
 ("Match1 && Match2", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3740, __PRETTY_FUNCTION__));

// Make sure our matches pair up.
if (!Match1->formsPairWith(*Match2))
  return SDValue();

// All the remains is to make sure C is an AND with the same input, that masks
// out the bits that are being shuffled.
if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
    C.getOperand(0) != Match1->Op)
  return SDValue();

uint64_t Mask = C.getConstantOperandVal(1);

static const uint64_t BitmanipMasks[] = {
    0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
    0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
};

unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
unsigned MaskIdx = Log2_32(Match1->ShAmt);
uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);

if (Mask != ExpMask)
  return SDValue();

SDLoc DL(Op);
return DAG.getNode(
    RISCVISD::SHFLI, DL, VT, Match1->Op,
    DAG.getTargetConstant(Match1->ShAmt, DL, Subtarget.getXLenVT()));
3770}

3772// Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
3773// non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
3774// Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
3775// not undo itself, but they are redundant.
3776static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
unsigned ShAmt1 = N->getConstantOperandVal(1);
SDValue Src = N->getOperand(0);

if (Src.getOpcode() != N->getOpcode())
  return SDValue();

unsigned ShAmt2 = Src.getConstantOperandVal(1);
Src = Src.getOperand(0);

unsigned CombinedShAmt;
if (N->getOpcode() == RISCVISD::GORCI || N->getOpcode() == RISCVISD::GORCIW)
  CombinedShAmt = ShAmt1 | ShAmt2;
else
  CombinedShAmt = ShAmt1 ^ ShAmt2;

if (CombinedShAmt == 0)
  return Src;

SDLoc DL(N);
return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), Src,
                   DAG.getTargetConstant(CombinedShAmt, DL,
                                         N->getOperand(1).getValueType()));
3799}

3801SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;

switch (N->getOpcode()) {
default:
  break;
case RISCVISD::SplitF64: {
  SDValue Op0 = N->getOperand(0);
  // If the input to SplitF64 is just BuildPairF64 then the operation is
  // redundant. Instead, use BuildPairF64's operands directly.
  if (Op0->getOpcode() == RISCVISD::BuildPairF64)
    return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));

  SDLoc DL(N);

  // It's cheaper to materialise two 32-bit integers than to load a double
  // from the constant pool and transfer it to integer registers through the
  // stack.
  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
    APInt V = C->getValueAPF().bitcastToAPInt();
    SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
    SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
    return DCI.CombineTo(N, Lo, Hi);
  }

  // This is a target-specific version of a DAGCombine performed in
  // DAGCombiner::visitBITCAST. It performs the equivalent of:
  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
  if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
      !Op0.getNode()->hasOneUse())
    break;
  SDValue NewSplitF64 =
      DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
                  Op0.getOperand(0));
  SDValue Lo = NewSplitF64.getValue(0);
  SDValue Hi = NewSplitF64.getValue(1);
  APInt SignBit = APInt::getSignMask(32);
  if (Op0.getOpcode() == ISD::FNEG) {
    SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
                                DAG.getConstant(SignBit, DL, MVT::i32));
    return DCI.CombineTo(N, Lo, NewHi);
  }
  assert(Op0.getOpcode() == ISD::FABS)((Op0.getOpcode() == ISD::FABS) ? static_cast<void> (0)
 : __assert_fail ("Op0.getOpcode() == ISD::FABS", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3845, __PRETTY_FUNCTION__));
  SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
                              DAG.getConstant(~SignBit, DL, MVT::i32));
  return DCI.CombineTo(N, Lo, NewHi);
}
case RISCVISD::SLLW:
case RISCVISD::SRAW:
case RISCVISD::SRLW:
case RISCVISD::ROLW:
case RISCVISD::RORW: {
  // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
  APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
  if (SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI) ||
      SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)) {
    if (N->getOpcode() != ISD::DELETED_NODE)
      DCI.AddToWorklist(N);
    return SDValue(N, 0);
  }
  break;
}
case RISCVISD::FSL:
case RISCVISD::FSR: {
  // Only the lower log2(Bitwidth)+1 bits of the the shift amount are read.
  SDValue ShAmt = N->getOperand(2);
  unsigned BitWidth = ShAmt.getValueSizeInBits();
  assert(isPowerOf2_32(BitWidth) && "Unexpected bit width")((isPowerOf2_32(BitWidth) && "Unexpected bit width") ?
 static_cast<void> (0) : __assert_fail ("isPowerOf2_32(BitWidth) && \"Unexpected bit width\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3873, __PRETTY_FUNCTION__));
  APInt ShAmtMask(BitWidth, (BitWidth * 2) - 1);
  if (SimplifyDemandedBits(ShAmt, ShAmtMask, DCI)) {
    if (N->getOpcode() != ISD::DELETED_NODE)
      DCI.AddToWorklist(N);
    return SDValue(N, 0);
  }
  break;
}
case RISCVISD::FSLW:
case RISCVISD::FSRW: {
  // Only the lower 32 bits of Values and lower 6 bits of shift amount are
  // read.
  SDValue Op0 = N->getOperand(0);
  SDValue Op1 = N->getOperand(1);
  SDValue ShAmt = N->getOperand(2);
  APInt OpMask = APInt::getLowBitsSet(Op0.getValueSizeInBits(), 32);
  APInt ShAmtMask = APInt::getLowBitsSet(ShAmt.getValueSizeInBits(), 6);
  if (SimplifyDemandedBits(Op0, OpMask, DCI) ||
      SimplifyDemandedBits(Op1, OpMask, DCI) ||
      SimplifyDemandedBits(ShAmt, ShAmtMask, DCI)) {
    if (N->getOpcode() != ISD::DELETED_NODE)
      DCI.AddToWorklist(N);
    return SDValue(N, 0);
  }
  break;
}
case RISCVISD::GREVIW:
case RISCVISD::GORCIW: {
  // Only the lower 32 bits of the first operand are read
  SDValue Op0 = N->getOperand(0);
  APInt Mask = APInt::getLowBitsSet(Op0.getValueSizeInBits(), 32);
  if (SimplifyDemandedBits(Op0, Mask, DCI)) {
    if (N->getOpcode() != ISD::DELETED_NODE)
      DCI.AddToWorklist(N);
    return SDValue(N, 0);
  }

  return combineGREVI_GORCI(N, DCI.DAG);
}
case RISCVISD::FMV_X_ANYEXTW_RV64: {
  SDLoc DL(N);
  SDValue Op0 = N->getOperand(0);
  // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
  // conversion is unnecessary and can be replaced with an ANY_EXTEND
  // of the FMV_W_X_RV64 operand.
  if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
    assert(Op0.getOperand(0).getValueType() == MVT::i64 &&((Op0.getOperand(0).getValueType() == MVT::i64 && "Unexpected value type!"
) ? static_cast<void> (0) : __assert_fail ("Op0.getOperand(0).getValueType() == MVT::i64 && \"Unexpected value type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3921, __PRETTY_FUNCTION__))
           "Unexpected value type!")((Op0.getOperand(0).getValueType() == MVT::i64 && "Unexpected value type!"
) ? static_cast<void> (0) : __assert_fail ("Op0.getOperand(0).getValueType() == MVT::i64 && \"Unexpected value type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3921, __PRETTY_FUNCTION__));
    return Op0.getOperand(0);
  }

  // This is a target-specific version of a DAGCombine performed in
  // DAGCombiner::visitBITCAST. It performs the equivalent of:
  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
  if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
      !Op0.getNode()->hasOneUse())
    break;
  SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64,
                               Op0.getOperand(0));
  APInt SignBit = APInt::getSignMask(32).sext(64);
  if (Op0.getOpcode() == ISD::FNEG)
    return DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
                       DAG.getConstant(SignBit, DL, MVT::i64));

  assert(Op0.getOpcode() == ISD::FABS)((Op0.getOpcode() == ISD::FABS) ? static_cast<void> (0)
 : __assert_fail ("Op0.getOpcode() == ISD::FABS", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 3939, __PRETTY_FUNCTION__));
  return DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
                     DAG.getConstant(~SignBit, DL, MVT::i64));
}
case RISCVISD::GREVI:
case RISCVISD::GORCI:
  return combineGREVI_GORCI(N, DCI.DAG);
case ISD::OR:
  if (auto GREV = combineORToGREV(SDValue(N, 0), DCI.DAG, Subtarget))
    return GREV;
  if (auto GORC = combineORToGORC(SDValue(N, 0), DCI.DAG, Subtarget))
    return GORC;
  if (auto SHFL = combineORToSHFL(SDValue(N, 0), DCI.DAG, Subtarget))
    return SHFL;
  break;
case RISCVISD::SELECT_CC: {
  // Transform
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  auto CCVal = static_cast<ISD::CondCode>(N->getConstantOperandVal(2));
  if (!ISD::isIntEqualitySetCC(CCVal))
    break;

  // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
  //      (select_cc X, Y, lt, trueV, falseV)
  // Sometimes the setcc is introduced after select_cc has been formed.
  if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
      LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
    // If we're looking for eq 0 instead of ne 0, we need to invert the
    // condition.
    bool Invert = CCVal == ISD::SETEQ;
    CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
    if (Invert)
      CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());

    RHS = LHS.getOperand(1);
    LHS = LHS.getOperand(0);
    normaliseSetCC(LHS, RHS, CCVal);

    SDLoc DL(N);
    SDValue TargetCC = DAG.getConstant(CCVal, DL, Subtarget.getXLenVT());
    return DAG.getNode(
        RISCVISD::SELECT_CC, DL, N->getValueType(0),
        {LHS, RHS, TargetCC, N->getOperand(3), N->getOperand(4)});
  }

  // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
  //      (select_cc X, Y, eq/ne, trueV, falseV)
  if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
    return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
                       {LHS.getOperand(0), LHS.getOperand(1),
                        N->getOperand(2), N->getOperand(3),
                        N->getOperand(4)});
  // (select_cc X, 1, setne, trueV, falseV) ->
  // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
  // This can occur when legalizing some floating point comparisons.
  APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
  if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
    SDLoc DL(N);
    CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
    SDValue TargetCC = DAG.getConstant(CCVal, DL, Subtarget.getXLenVT());
    RHS = DAG.getConstant(0, DL, LHS.getValueType());
    return DAG.getNode(
        RISCVISD::SELECT_CC, DL, N->getValueType(0),
        {LHS, RHS, TargetCC, N->getOperand(3), N->getOperand(4)});
  }

  break;
}
case ISD::SETCC: {
  // (setcc X, 1, setne) -> (setcc X, 0, seteq) if we can prove X is 0/1.
  // Comparing with 0 may allow us to fold into bnez/beqz.
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  if (LHS.getValueType().isScalableVector())
    break;
  auto CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
  APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
  if (isOneConstant(RHS) && ISD::isIntEqualitySetCC(CC) &&
      DAG.MaskedValueIsZero(LHS, Mask)) {
    SDLoc DL(N);
    SDValue Zero = DAG.getConstant(0, DL, LHS.getValueType());
    CC = ISD::getSetCCInverse(CC, LHS.getValueType());
    return DAG.getSetCC(DL, N->getValueType(0), LHS, Zero, CC);
  }
  break;
}
case ISD::FCOPYSIGN: {
  EVT VT = N->getValueType(0);
  if (!VT.isVector())
    break;
  // There is a form of VFSGNJ which injects the negated sign of its second
  // operand. Try and bubble any FNEG up after the extend/round to produce
  // this optimized pattern. Avoid modifying cases where FP_ROUND and
  // TRUNC=1.
  SDValue In2 = N->getOperand(1);
  // Avoid cases where the extend/round has multiple uses, as duplicating
  // those is typically more expensive than removing a fneg.
  if (!In2.hasOneUse())
    break;
  if (In2.getOpcode() != ISD::FP_EXTEND &&
      (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
    break;
  In2 = In2.getOperand(0);
  if (In2.getOpcode() != ISD::FNEG)
    break;
  SDLoc DL(N);
  SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
  return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
                     DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
}
}

return SDValue();
4053}

4055bool RISCVTargetLowering::isDesirableToCommuteWithShift(
  const SDNode *N, CombineLevel Level) const {
// The following folds are only desirable if `(OP _, c1 << c2)` can be
// materialised in fewer instructions than `(OP _, c1)`:
//
//   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
//   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
SDValue N0 = N->getOperand(0);
EVT Ty = N0.getValueType();
if (Ty.isScalarInteger() &&
    (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
  auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
  auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (C1 && C2) {
    const APInt &C1Int = C1->getAPIntValue();
    APInt ShiftedC1Int = C1Int << C2->getAPIntValue();

    // We can materialise `c1 << c2` into an add immediate, so it's "free",
    // and the combine should happen, to potentially allow further combines
    // later.
    if (ShiftedC1Int.getMinSignedBits() <= 64 &&
        isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
      return true;

    // We can materialise `c1` in an add immediate, so it's "free", and the
    // combine should be prevented.
    if (C1Int.getMinSignedBits() <= 64 &&
        isLegalAddImmediate(C1Int.getSExtValue()))
      return false;

    // Neither constant will fit into an immediate, so find materialisation
    // costs.
    int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
                                            Subtarget.is64Bit());
    int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
        ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit());

    // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
    // combine should be prevented.
    if (C1Cost < ShiftedC1Cost)
      return false;
  }
}
return true;
4099}

4101bool RISCVTargetLowering::targetShrinkDemandedConstant(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  TargetLoweringOpt &TLO) const {
// Delay this optimization as late as possible.
if (!TLO.LegalOps)
  return false;

EVT VT = Op.getValueType();
if (VT.isVector())
  return false;

// Only handle AND for now.
if (Op.getOpcode() != ISD::AND)
  return false;

ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
if (!C)
  return false;

const APInt &Mask = C->getAPIntValue();

// Clear all non-demanded bits initially.
APInt ShrunkMask = Mask & DemandedBits;

// If the shrunk mask fits in sign extended 12 bits, let the target
// independent code apply it.
if (ShrunkMask.isSignedIntN(12))
  return false;

// Try to make a smaller immediate by setting undemanded bits.

// We need to be able to make a negative number through a combination of mask
// and undemanded bits.
APInt ExpandedMask = Mask | ~DemandedBits;
if (!ExpandedMask.isNegative())
  return false;

// What is the fewest number of bits we need to represent the negative number.
unsigned MinSignedBits = ExpandedMask.getMinSignedBits();

// Try to make a 12 bit negative immediate. If that fails try to make a 32
// bit negative immediate unless the shrunk immediate already fits in 32 bits.
APInt NewMask = ShrunkMask;
if (MinSignedBits <= 12)
  NewMask.setBitsFrom(11);
else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
  NewMask.setBitsFrom(31);
else
  return false;

// Sanity check that our new mask is a subset of the demanded mask.
assert(NewMask.isSubsetOf(ExpandedMask))((NewMask.isSubsetOf(ExpandedMask)) ? static_cast<void>
 (0) : __assert_fail ("NewMask.isSubsetOf(ExpandedMask)", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4152, __PRETTY_FUNCTION__));

// If we aren't changing the mask, just return true to keep it and prevent
// the caller from optimizing.
if (NewMask == Mask)
  return true;

// Replace the constant with the new mask.
SDLoc DL(Op);
SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
return TLO.CombineTo(Op, NewOp);
4164}

4166void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
                                                      KnownBits &Known,
                                                      const APInt &DemandedElts,
                                                      const SelectionDAG &DAG,
                                                      unsigned Depth) const {
unsigned BitWidth = Known.getBitWidth();
unsigned Opc = Op.getOpcode();
assert((Opc >= ISD::BUILTIN_OP_END ||(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
 || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4178, __PRETTY_FUNCTION__))
        Opc == ISD::INTRINSIC_WO_CHAIN ||(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
 || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4178, __PRETTY_FUNCTION__))
        Opc == ISD::INTRINSIC_W_CHAIN ||(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
 || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4178, __PRETTY_FUNCTION__))
        Opc == ISD::INTRINSIC_VOID) &&(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
 || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4178, __PRETTY_FUNCTION__))
       "Should use MaskedValueIsZero if you don't know whether Op"(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
 || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4178, __PRETTY_FUNCTION__))
       " is a target node!")(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
 || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4178, __PRETTY_FUNCTION__));

Known.resetAll();
switch (Opc) {
default: break;
case RISCVISD::REMUW: {
  KnownBits Known2;
  Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
  // We only care about the lower 32 bits.
  Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
  // Restore the original width by sign extending.
  Known = Known.sext(BitWidth);
  break;
}
case RISCVISD::DIVUW: {
  KnownBits Known2;
  Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
  // We only care about the lower 32 bits.
  Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
  // Restore the original width by sign extending.
  Known = Known.sext(BitWidth);
  break;
}
case RISCVISD::READ_VLENB:
  // We assume VLENB is at least 8 bytes.
  // FIXME: The 1.0 draft spec defines minimum VLEN as 128 bits.
  Known.Zero.setLowBits(3);
  break;
}
4209}

4211unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
  unsigned Depth) const {
switch (Op.getOpcode()) {
default:
  break;
case RISCVISD::SLLW:
case RISCVISD::SRAW:
case RISCVISD::SRLW:
case RISCVISD::DIVW:
case RISCVISD::DIVUW:
case RISCVISD::REMUW:
case RISCVISD::ROLW:
case RISCVISD::RORW:
case RISCVISD::GREVIW:
case RISCVISD::GORCIW:
case RISCVISD::FSLW:
case RISCVISD::FSRW:
  // TODO: As the result is sign-extended, this is conservatively correct. A
  // more precise answer could be calculated for SRAW depending on known
  // bits in the shift amount.
  return 33;
case RISCVISD::SHFLI: {
  // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
  // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
  // will stay within the upper 32 bits. If there were more than 32 sign bits
  // before there will be at least 33 sign bits after.
  if (Op.getValueType() == MVT::i64 &&
      (Op.getConstantOperandVal(1) & 0x10) == 0) {
    unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
    if (Tmp > 32)
      return 33;
  }
  break;
}
case RISCVISD::VMV_X_S:
  // The number of sign bits of the scalar result is computed by obtaining the
  // element type of the input vector operand, subtracting its width from the
  // XLEN, and then adding one (sign bit within the element type). If the
  // element type is wider than XLen, the least-significant XLEN bits are
  // taken.
  if (Op.getOperand(0).getScalarValueSizeInBits() > Subtarget.getXLen())
    return 1;
  return Subtarget.getXLen() - Op.getOperand(0).getScalarValueSizeInBits() + 1;
}

return 1;
4258}

4260static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
                                                MachineBasicBlock *BB) {
assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction")((MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::ReadCycleWide && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4262, __PRETTY_FUNCTION__));

// To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
// Should the count have wrapped while it was being read, we need to try
// again.
// ...
// read:
// rdcycleh x3 # load high word of cycle
// rdcycle  x2 # load low word of cycle
// rdcycleh x4 # load high word of cycle
// bne x3, x4, read # check if high word reads match, otherwise try again
// ...

MachineFunction &MF = *BB->getParent();
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = ++BB->getIterator();

MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
MF.insert(It, LoopMBB);

MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
MF.insert(It, DoneMBB);

// Transfer the remainder of BB and its successor edges to DoneMBB.
DoneMBB->splice(DoneMBB->begin(), BB,
                std::next(MachineBasicBlock::iterator(MI)), BB->end());
DoneMBB->transferSuccessorsAndUpdatePHIs(BB);

BB->addSuccessor(LoopMBB);

MachineRegisterInfo &RegInfo = MF.getRegInfo();
Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
Register LoReg = MI.getOperand(0).getReg();
Register HiReg = MI.getOperand(1).getReg();
DebugLoc DL = MI.getDebugLoc();

const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
    .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
    .addReg(RISCV::X0);
BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
    .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
    .addReg(RISCV::X0);
BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
    .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
    .addReg(RISCV::X0);

BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
    .addReg(HiReg)
    .addReg(ReadAgainReg)
    .addMBB(LoopMBB);

LoopMBB->addSuccessor(LoopMBB);
LoopMBB->addSuccessor(DoneMBB);

MI.eraseFromParent();

return DoneMBB;
4320}

4322static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
                                           MachineBasicBlock *BB) {
assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction")((MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::SplitF64Pseudo && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4324, __PRETTY_FUNCTION__));

MachineFunction &MF = *BB->getParent();
DebugLoc DL = MI.getDebugLoc();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
Register LoReg = MI.getOperand(0).getReg();
Register HiReg = MI.getOperand(1).getReg();
Register SrcReg = MI.getOperand(2).getReg();
const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);

TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
                        RI);
MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
MachineMemOperand *MMOLo =
    MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
MachineMemOperand *MMOHi = MF.getMachineMemOperand(
    MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
    .addFrameIndex(FI)
    .addImm(0)
    .addMemOperand(MMOLo);
BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
    .addFrameIndex(FI)
    .addImm(4)
    .addMemOperand(MMOHi);
MI.eraseFromParent(); // The pseudo instruction is gone now.
return BB;
4353}

4355static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
                                               MachineBasicBlock *BB) {
assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&((MI.getOpcode() == RISCV::BuildPairF64Pseudo && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::BuildPairF64Pseudo && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4358, __PRETTY_FUNCTION__))
       "Unexpected instruction")((MI.getOpcode() == RISCV::BuildPairF64Pseudo && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::BuildPairF64Pseudo && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4358, __PRETTY_FUNCTION__));

MachineFunction &MF = *BB->getParent();
DebugLoc DL = MI.getDebugLoc();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
Register DstReg = MI.getOperand(0).getReg();
Register LoReg = MI.getOperand(1).getReg();
Register HiReg = MI.getOperand(2).getReg();
const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);

MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
MachineMemOperand *MMOLo =
    MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
MachineMemOperand *MMOHi = MF.getMachineMemOperand(
    MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
    .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
    .addFrameIndex(FI)
    .addImm(0)
    .addMemOperand(MMOLo);
BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
    .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
    .addFrameIndex(FI)
    .addImm(4)
    .addMemOperand(MMOHi);
TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
MI.eraseFromParent(); // The pseudo instruction is gone now.
return BB;
4388}

4390static bool isSelectPseudo(MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
  return false;
case RISCV::Select_GPR_Using_CC_GPR:
case RISCV::Select_FPR16_Using_CC_GPR:
case RISCV::Select_FPR32_Using_CC_GPR:
case RISCV::Select_FPR64_Using_CC_GPR:
  return true;
}
4400}

4402static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
                                         MachineBasicBlock *BB) {
// To "insert" Select_* instructions, we actually have to insert the triangle
// control-flow pattern.  The incoming instructions know the destination vreg
// to set, the condition code register to branch on, the true/false values to
// select between, and the condcode to use to select the appropriate branch.
//
// We produce the following control flow:
//     HeadMBB
//     |  \
//     |  IfFalseMBB
//     | /
//    TailMBB
//
// When we find a sequence of selects we attempt to optimize their emission
// by sharing the control flow. Currently we only handle cases where we have
// multiple selects with the exact same condition (same LHS, RHS and CC).
// The selects may be interleaved with other instructions if the other
// instructions meet some requirements we deem safe:
// - They are debug instructions. Otherwise,
// - They do not have side-effects, do not access memory and their inputs do
//   not depend on the results of the select pseudo-instructions.
// The TrueV/FalseV operands of the selects cannot depend on the result of
// previous selects in the sequence.
// These conditions could be further relaxed. See the X86 target for a
// related approach and more information.
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());

SmallVector<MachineInstr *, 4> SelectDebugValues;
SmallSet<Register, 4> SelectDests;
SelectDests.insert(MI.getOperand(0).getReg());

MachineInstr *LastSelectPseudo = &MI;

for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
     SequenceMBBI != E; ++SequenceMBBI) {
  if (SequenceMBBI->isDebugInstr())
    continue;
  else if (isSelectPseudo(*SequenceMBBI)) {
    if (SequenceMBBI->getOperand(1).getReg() != LHS ||
        SequenceMBBI->getOperand(2).getReg() != RHS ||
        SequenceMBBI->getOperand(3).getImm() != CC ||
        SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
        SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
      break;
    LastSelectPseudo = &*SequenceMBBI;
    SequenceMBBI->collectDebugValues(SelectDebugValues);
    SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
  } else {
    if (SequenceMBBI->hasUnmodeledSideEffects() ||
        SequenceMBBI->mayLoadOrStore())
      break;
    if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
          return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
        }))
      break;
  }
}

const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
const BasicBlock *LLVM_BB = BB->getBasicBlock();
DebugLoc DL = MI.getDebugLoc();
MachineFunction::iterator I = ++BB->getIterator();

MachineBasicBlock *HeadMBB = BB;
MachineFunction *F = BB->getParent();
MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);

F->insert(I, IfFalseMBB);
F->insert(I, TailMBB);

// Transfer debug instructions associated with the selects to TailMBB.
for (MachineInstr *DebugInstr : SelectDebugValues) {
  TailMBB->push_back(DebugInstr->removeFromParent());
}

// Move all instructions after the sequence to TailMBB.
TailMBB->splice(TailMBB->end(), HeadMBB,
                std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
// Update machine-CFG edges by transferring all successors of the current
// block to the new block which will contain the Phi nodes for the selects.
TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
// Set the successors for HeadMBB.
HeadMBB->addSuccessor(IfFalseMBB);
HeadMBB->addSuccessor(TailMBB);

// Insert appropriate branch.
unsigned Opcode = getBranchOpcodeForIntCondCode(CC);

BuildMI(HeadMBB, DL, TII.get(Opcode))
  .addReg(LHS)
  .addReg(RHS)
  .addMBB(TailMBB);

// IfFalseMBB just falls through to TailMBB.
IfFalseMBB->addSuccessor(TailMBB);

// Create PHIs for all of the select pseudo-instructions.
auto SelectMBBI = MI.getIterator();
auto SelectEnd = std::next(LastSelectPseudo->getIterator());
auto InsertionPoint = TailMBB->begin();
while (SelectMBBI != SelectEnd) {
  auto Next = std::next(SelectMBBI);
  if (isSelectPseudo(*SelectMBBI)) {
    // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
    BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
            TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
        .addReg(SelectMBBI->getOperand(4).getReg())
        .addMBB(HeadMBB)
        .addReg(SelectMBBI->getOperand(5).getReg())
        .addMBB(IfFalseMBB);
    SelectMBBI->eraseFromParent();
  }
  SelectMBBI = Next;
}

F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
return TailMBB;
4523}

4525static MachineBasicBlock *addVSetVL(MachineInstr &MI, MachineBasicBlock *BB,
                                  int VLIndex, unsigned SEWIndex,
                                  RISCVVLMUL VLMul, bool ForceTailAgnostic) {
MachineFunction &MF = *BB->getParent();
DebugLoc DL = MI.getDebugLoc();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();

unsigned SEW = MI.getOperand(SEWIndex).getImm();
assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW")((RISCVVType::isValidSEW(SEW) && "Unexpected SEW") ? static_cast
<void> (0) : __assert_fail ("RISCVVType::isValidSEW(SEW) && \"Unexpected SEW\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4533, __PRETTY_FUNCTION__));
RISCVVSEW ElementWidth = static_cast<RISCVVSEW>(Log2_32(SEW / 8));

MachineRegisterInfo &MRI = MF.getRegInfo();

auto BuildVSETVLI = [&]() {
  if (VLIndex >= 0) {
    Register DestReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
    Register VLReg = MI.getOperand(VLIndex).getReg();

    // VL might be a compile time constant, but isel would have to put it
    // in a register. See if VL comes from an ADDI X0, imm.
    if (VLReg.isVirtual()) {
      MachineInstr *Def = MRI.getVRegDef(VLReg);
      if (Def && Def->getOpcode() == RISCV::ADDI &&
          Def->getOperand(1).getReg() == RISCV::X0 &&
          Def->getOperand(2).isImm()) {
        uint64_t Imm = Def->getOperand(2).getImm();
        // VSETIVLI allows a 5-bit zero extended immediate.
        if (isUInt<5>(Imm))
          return BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVSETIVLI))
              .addReg(DestReg, RegState::Define | RegState::Dead)
              .addImm(Imm);
      }
    }

    return BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVSETVLI))
        .addReg(DestReg, RegState::Define | RegState::Dead)
        .addReg(VLReg);
  }

  // With no VL operator in the pseudo, do not modify VL (rd = X0, rs1 = X0).
  return BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVSETVLI))
      .addReg(RISCV::X0, RegState::Define | RegState::Dead)
      .addReg(RISCV::X0, RegState::Kill);
};

MachineInstrBuilder MIB = BuildVSETVLI();

// Default to tail agnostic unless the destination is tied to a source. In
// that case the user would have some control over the tail values. The tail
// policy is also ignored on instructions that only update element 0 like
// vmv.s.x or reductions so use agnostic there to match the common case.
// FIXME: This is conservatively correct, but we might want to detect that
// the input is undefined.
bool TailAgnostic = true;
unsigned UseOpIdx;
if (!ForceTailAgnostic && MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
  TailAgnostic = false;
  // If the tied operand is an IMPLICIT_DEF we can keep TailAgnostic.
  const MachineOperand &UseMO = MI.getOperand(UseOpIdx);
  MachineInstr *UseMI = MRI.getVRegDef(UseMO.getReg());
  if (UseMI && UseMI->isImplicitDef())
    TailAgnostic = true;
}

// For simplicity we reuse the vtype representation here.
MIB.addImm(RISCVVType::encodeVTYPE(VLMul, ElementWidth,
                                   /*TailAgnostic*/ TailAgnostic,
                                   /*MaskAgnostic*/ false));

// Remove (now) redundant operands from pseudo
if (VLIndex >= 0) {
  MI.getOperand(VLIndex).setReg(RISCV::NoRegister);
  MI.getOperand(VLIndex).setIsKill(false);
}

return BB;
4601}

4603MachineBasicBlock *
4604RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                                               MachineBasicBlock *BB) const {
uint64_t TSFlags = MI.getDesc().TSFlags;

if (TSFlags & RISCVII::HasSEWOpMask) {
  unsigned NumOperands = MI.getNumExplicitOperands();
  int VLIndex = (TSFlags & RISCVII::HasVLOpMask) ? NumOperands - 2 : -1;
  unsigned SEWIndex = NumOperands - 1;
  bool ForceTailAgnostic = TSFlags & RISCVII::ForceTailAgnosticMask;

  RISCVVLMUL VLMul = static_cast<RISCVVLMUL>((TSFlags & RISCVII::VLMulMask) >>
                                             RISCVII::VLMulShift);
  return addVSetVL(MI, BB, VLIndex, SEWIndex, VLMul, ForceTailAgnostic);
}

switch (MI.getOpcode()) {
default:
  llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4621);
case RISCV::ReadCycleWide:
  assert(!Subtarget.is64Bit() &&((!Subtarget.is64Bit() && "ReadCycleWrite is only to be used on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"ReadCycleWrite is only to be used on riscv32\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4624, __PRETTY_FUNCTION__))
         "ReadCycleWrite is only to be used on riscv32")((!Subtarget.is64Bit() && "ReadCycleWrite is only to be used on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"ReadCycleWrite is only to be used on riscv32\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4624, __PRETTY_FUNCTION__));
  return emitReadCycleWidePseudo(MI, BB);
case RISCV::Select_GPR_Using_CC_GPR:
case RISCV::Select_FPR16_Using_CC_GPR:
case RISCV::Select_FPR32_Using_CC_GPR:
case RISCV::Select_FPR64_Using_CC_GPR:
  return emitSelectPseudo(MI, BB);
case RISCV::BuildPairF64Pseudo:
  return emitBuildPairF64Pseudo(MI, BB);
case RISCV::SplitF64Pseudo:
  return emitSplitF64Pseudo(MI, BB);
}
4636}

4638// Calling Convention Implementation.
4639// The expectations for frontend ABI lowering vary from target to target.
4640// Ideally, an LLVM frontend would be able to avoid worrying about many ABI
4641// details, but this is a longer term goal. For now, we simply try to keep the
4642// role of the frontend as simple and well-defined as possible. The rules can
4643// be summarised as:
4644// * Never split up large scalar arguments. We handle them here.
4645// * If a hardfloat calling convention is being used, and the struct may be
4646// passed in a pair of registers (fp+fp, int+fp), and both registers are
4647// available, then pass as two separate arguments. If either the GPRs or FPRs
4648// are exhausted, then pass according to the rule below.
4649// * If a struct could never be passed in registers or directly in a stack
4650// slot (as it is larger than 2*XLEN and the floating point rules don't
4651// apply), then pass it using a pointer with the byval attribute.
4652// * If a struct is less than 2*XLEN, then coerce to either a two-element
4653// word-sized array or a 2*XLEN scalar (depending on alignment).
4654// * The frontend can determine whether a struct is returned by reference or
4655// not based on its size and fields. If it will be returned by reference, the
4656// frontend must modify the prototype so a pointer with the sret annotation is
4657// passed as the first argument. This is not necessary for large scalar
4658// returns.
4659// * Struct return values and varargs should be coerced to structs containing
4660// register-size fields in the same situations they would be for fixed
4661// arguments.

4663static const MCPhysReg ArgGPRs[] = {
RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
4666};
4667static const MCPhysReg ArgFPR16s[] = {
RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
4670};
4671static const MCPhysReg ArgFPR32s[] = {
RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
4674};
4675static const MCPhysReg ArgFPR64s[] = {
RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
4678};
4679// This is an interim calling convention and it may be changed in the future.
4680static const MCPhysReg ArgVRs[] = {
  RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
  RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
  RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
4684static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
                                   RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
                                   RISCV::V20M2, RISCV::V22M2};
4687static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
                                   RISCV::V20M4};
4689static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};

4691// Pass a 2*XLEN argument that has been split into two XLEN values through
4692// registers or the stack as necessary.
4693static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
                              ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
                              MVT ValVT2, MVT LocVT2,
                              ISD::ArgFlagsTy ArgFlags2) {
unsigned XLenInBytes = XLen / 8;
if (Register Reg = State.AllocateReg(ArgGPRs)) {
  // At least one half can be passed via register.
  State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
                                   VA1.getLocVT(), CCValAssign::Full));
} else {
  // Both halves must be passed on the stack, with proper alignment.
  Align StackAlign =
      std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
  State.addLoc(
      CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
                          State.AllocateStack(XLenInBytes, StackAlign),
                          VA1.getLocVT(), CCValAssign::Full));
  State.addLoc(CCValAssign::getMem(
      ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
      LocVT2, CCValAssign::Full));
  return false;
}

if (Register Reg = State.AllocateReg(ArgGPRs)) {
  // The second half can also be passed via register.
  State.addLoc(
      CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
} else {
  // The second half is passed via the stack, without additional alignment.
  State.addLoc(CCValAssign::getMem(
      ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
      LocVT2, CCValAssign::Full));
}

return false;
4728}

4730// Implements the RISC-V calling convention. Returns true upon failure.
4731static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
                   MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
                   ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
                   bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
                   Optional<unsigned> FirstMaskArgument) {
unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
assert(XLen == 32 || XLen == 64)((XLen == 32 || XLen == 64) ? static_cast<void> (0) : __assert_fail
 ("XLen == 32 || XLen == 64", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4737, __PRETTY_FUNCTION__));
MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;

// Any return value split in to more than two values can't be returned
// directly.
if (IsRet && ValNo > 1)
  return true;

// UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
// variadic argument, or if no F16/F32 argument registers are available.
bool UseGPRForF16_F32 = true;
// UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
// variadic argument, or if no F64 argument registers are available.
bool UseGPRForF64 = true;

switch (ABI) {
default:
  llvm_unreachable("Unexpected ABI")::llvm::llvm_unreachable_internal("Unexpected ABI", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4754);
case RISCVABI::ABI_ILP32:
case RISCVABI::ABI_LP64:
  break;
case RISCVABI::ABI_ILP32F:
case RISCVABI::ABI_LP64F:
  UseGPRForF16_F32 = !IsFixed;
  break;
case RISCVABI::ABI_ILP32D:
case RISCVABI::ABI_LP64D:
  UseGPRForF16_F32 = !IsFixed;
  UseGPRForF64 = !IsFixed;
  break;
}

// FPR16, FPR32, and FPR64 alias each other.
if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
  UseGPRForF16_F32 = true;
  UseGPRForF64 = true;
}

// From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
// similar local variables rather than directly checking against the target
// ABI.

if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
  LocVT = XLenVT;
  LocInfo = CCValAssign::BCvt;
} else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
  LocVT = MVT::i64;
  LocInfo = CCValAssign::BCvt;
}

// If this is a variadic argument, the RISC-V calling convention requires
// that it is assigned an 'even' or 'aligned' register if it has 8-byte
// alignment (RV32) or 16-byte alignment (RV64). An aligned register should
// be used regardless of whether the original argument was split during
// legalisation or not. The argument will not be passed by registers if the
// original type is larger than 2*XLEN, so the register alignment rule does
// not apply.
unsigned TwoXLenInBytes = (2 * XLen) / 8;
if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
    DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
  unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
  // Skip 'odd' register if necessary.
  if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
    State.AllocateReg(ArgGPRs);
}

SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
    State.getPendingArgFlags();

assert(PendingLocs.size() == PendingArgFlags.size() &&((PendingLocs.size() == PendingArgFlags.size() && "PendingLocs and PendingArgFlags out of sync"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() == PendingArgFlags.size() && \"PendingLocs and PendingArgFlags out of sync\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4808, __PRETTY_FUNCTION__))
       "PendingLocs and PendingArgFlags out of sync")((PendingLocs.size() == PendingArgFlags.size() && "PendingLocs and PendingArgFlags out of sync"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() == PendingArgFlags.size() && \"PendingLocs and PendingArgFlags out of sync\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4808, __PRETTY_FUNCTION__));

// Handle passing f64 on RV32D with a soft float ABI or when floating point
// registers are exhausted.
if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
  assert(!ArgFlags.isSplit() && PendingLocs.empty() &&((!ArgFlags.isSplit() && PendingLocs.empty() &&
 "Can't lower f64 if it is split") ? static_cast<void> (
0) : __assert_fail ("!ArgFlags.isSplit() && PendingLocs.empty() && \"Can't lower f64 if it is split\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4814, __PRETTY_FUNCTION__))
         "Can't lower f64 if it is split")((!ArgFlags.isSplit() && PendingLocs.empty() &&
 "Can't lower f64 if it is split") ? static_cast<void> (
0) : __assert_fail ("!ArgFlags.isSplit() && PendingLocs.empty() && \"Can't lower f64 if it is split\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4814, __PRETTY_FUNCTION__));
  // Depending on available argument GPRS, f64 may be passed in a pair of
  // GPRs, split between a GPR and the stack, or passed completely on the
  // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
  // cases.
  Register Reg = State.AllocateReg(ArgGPRs);
  LocVT = MVT::i32;
  if (!Reg) {
    unsigned StackOffset = State.AllocateStack(8, Align(8));
    State.addLoc(
        CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
    return false;
  }
  if (!State.AllocateReg(ArgGPRs))
    State.AllocateStack(4, Align(4));
  State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  return false;
}

// Split arguments might be passed indirectly, so keep track of the pending
// values.
if (ArgFlags.isSplit() || !PendingLocs.empty()) {
  LocVT = XLenVT;
  LocInfo = CCValAssign::Indirect;
  PendingLocs.push_back(
      CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
  PendingArgFlags.push_back(ArgFlags);
  if (!ArgFlags.isSplitEnd()) {
    return false;
  }
}

// If the split argument only had two elements, it should be passed directly
// in registers or on the stack.
if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
  assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()")((PendingLocs.size() == 2 && "Unexpected PendingLocs.size()"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() == 2 && \"Unexpected PendingLocs.size()\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4849, __PRETTY_FUNCTION__));
  // Apply the normal calling convention rules to the first half of the
  // split argument.
  CCValAssign VA = PendingLocs[0];
  ISD::ArgFlagsTy AF = PendingArgFlags[0];
  PendingLocs.clear();
  PendingArgFlags.clear();
  return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
                             ArgFlags);
}

// Allocate to a register if possible, or else a stack slot.
Register Reg;
if (ValVT == MVT::f16 && !UseGPRForF16_F32)
  Reg = State.AllocateReg(ArgFPR16s);
else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
  Reg = State.AllocateReg(ArgFPR32s);
else if (ValVT == MVT::f64 && !UseGPRForF64)
  Reg = State.AllocateReg(ArgFPR64s);
else if (ValVT.isScalableVector()) {
  const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
  if (RC == &RISCV::VRRegClass) {
    // Assign the first mask argument to V0.
    // This is an interim calling convention and it may be changed in the
    // future.
    if (FirstMaskArgument.hasValue() &&
        ValNo == FirstMaskArgument.getValue()) {
      Reg = State.AllocateReg(RISCV::V0);
    } else {
      Reg = State.AllocateReg(ArgVRs);
    }
  } else if (RC == &RISCV::VRM2RegClass) {
    Reg = State.AllocateReg(ArgVRM2s);
  } else if (RC == &RISCV::VRM4RegClass) {
    Reg = State.AllocateReg(ArgVRM4s);
  } else if (RC == &RISCV::VRM8RegClass) {
    Reg = State.AllocateReg(ArgVRM8s);
  } else {
    llvm_unreachable("Unhandled class register for ValueType")::llvm::llvm_unreachable_internal("Unhandled class register for ValueType"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4887);
  }
  if (!Reg) {
    LocInfo = CCValAssign::Indirect;
    // Try using a GPR to pass the address
    Reg = State.AllocateReg(ArgGPRs);
    LocVT = XLenVT;
  }
} else
  Reg = State.AllocateReg(ArgGPRs);
unsigned StackOffset =
    Reg ? 0 : State.AllocateStack(XLen / 8, Align(XLen / 8));

// If we reach this point and PendingLocs is non-empty, we must be at the
// end of a split argument that must be passed indirectly.
if (!PendingLocs.empty()) {
  assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()")((ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()"
) ? static_cast<void> (0) : __assert_fail ("ArgFlags.isSplitEnd() && \"Expected ArgFlags.isSplitEnd()\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4903, __PRETTY_FUNCTION__));
  assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()")((PendingLocs.size() > 2 && "Unexpected PendingLocs.size()"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() > 2 && \"Unexpected PendingLocs.size()\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4904, __PRETTY_FUNCTION__));

  for (auto &It : PendingLocs) {
    if (Reg)
      It.convertToReg(Reg);
    else
      It.convertToMem(StackOffset);
    State.addLoc(It);
  }
  PendingLocs.clear();
  PendingArgFlags.clear();
  return false;
}

assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||(((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT || (
TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector
())) && "Expected an XLenVT or scalable vector types at this stage"
) ? static_cast<void> (0) : __assert_fail ("(!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT || (TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector())) && \"Expected an XLenVT or scalable vector types at this stage\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4920, __PRETTY_FUNCTION__))
        (TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector())) &&(((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT || (
TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector
())) && "Expected an XLenVT or scalable vector types at this stage"
) ? static_cast<void> (0) : __assert_fail ("(!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT || (TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector())) && \"Expected an XLenVT or scalable vector types at this stage\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4920, __PRETTY_FUNCTION__))
       "Expected an XLenVT or scalable vector types at this stage")(((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT || (
TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector
())) && "Expected an XLenVT or scalable vector types at this stage"
) ? static_cast<void> (0) : __assert_fail ("(!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT || (TLI.getSubtarget().hasStdExtV() && ValVT.isScalableVector())) && \"Expected an XLenVT or scalable vector types at this stage\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4920, __PRETTY_FUNCTION__));

if (Reg) {
  State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
  return false;
}

// When a floating-point value is passed on the stack, no bit-conversion is
// needed.
if (ValVT.isFloatingPoint()) {
  LocVT = ValVT;
  LocInfo = CCValAssign::Full;
}
State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
return false;
4935}

4937template <typename ArgTy>
4938static Optional<unsigned> preAssignMask(const ArgTy &Args) {
for (const auto &ArgIdx : enumerate(Args)) {
  MVT ArgVT = ArgIdx.value().VT;
  if (ArgVT.isScalableVector() &&
      ArgVT.getVectorElementType().SimpleTy == MVT::i1)
    return ArgIdx.index();
}
return None;
4946}

4948void RISCVTargetLowering::analyzeInputArgs(
  MachineFunction &MF, CCState &CCInfo,
  const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
unsigned NumArgs = Ins.size();
FunctionType *FType = MF.getFunction().getFunctionType();

Optional<unsigned> FirstMaskArgument;
if (Subtarget.hasStdExtV())
  FirstMaskArgument = preAssignMask(Ins);

for (unsigned i = 0; i != NumArgs; ++i) {
  MVT ArgVT = Ins[i].VT;
  ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;

  Type *ArgTy = nullptr;
  if (IsRet)
    ArgTy = FType->getReturnType();
  else if (Ins[i].isOrigArg())
    ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());

  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
  if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
               ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
               FirstMaskArgument)) {
    LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "InputArg #" << i <<
 " has unhandled type " << EVT(ArgVT).getEVTString() <<
 '\n'; } } while (false)
                      << EVT(ArgVT).getEVTString() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "InputArg #" << i <<
 " has unhandled type " << EVT(ArgVT).getEVTString() <<
 '\n'; } } while (false);
    llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 4974);
  }
}
4977}

4979void RISCVTargetLowering::analyzeOutputArgs(
  MachineFunction &MF, CCState &CCInfo,
  const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
  CallLoweringInfo *CLI) const {
unsigned NumArgs = Outs.size();

Optional<unsigned> FirstMaskArgument;
if (Subtarget.hasStdExtV())
  FirstMaskArgument = preAssignMask(Outs);

for (unsigned i = 0; i != NumArgs; i++) {
  MVT ArgVT = Outs[i].VT;
  ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
  Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;

  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
  if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
               ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
               FirstMaskArgument)) {
    LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "OutputArg #" << i <<
 " has unhandled type " << EVT(ArgVT).getEVTString() <<
 "\n"; } } while (false)
                      << EVT(ArgVT).getEVTString() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "OutputArg #" << i <<
 " has unhandled type " << EVT(ArgVT).getEVTString() <<
 "\n"; } } while (false);
    llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5000);
  }
}
5003}

5005// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
5006// values.
5007static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
                                 const CCValAssign &VA, const SDLoc &DL) {
switch (VA.getLocInfo()) {
default:
  llvm_unreachable("Unexpected CCValAssign::LocInfo")::llvm::llvm_unreachable_internal("Unexpected CCValAssign::LocInfo"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5011);
case CCValAssign::Full:
  break;
case CCValAssign::BCvt:
  if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
    Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
  else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
    Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
  else
    Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
  break;
}
return Val;
5024}

5026// The caller is responsible for loading the full value if the argument is
5027// passed with CCValAssign::Indirect.
5028static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
                              const CCValAssign &VA, const SDLoc &DL,
                              const RISCVTargetLowering &TLI) {
MachineFunction &MF = DAG.getMachineFunction();
MachineRegisterInfo &RegInfo = MF.getRegInfo();
EVT LocVT = VA.getLocVT();
SDValue Val;
const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
Register VReg = RegInfo.createVirtualRegister(RC);
RegInfo.addLiveIn(VA.getLocReg(), VReg);
Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);

if (VA.getLocInfo() == CCValAssign::Indirect)
  return Val;

return convertLocVTToValVT(DAG, Val, VA, DL);
5044}

5046static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
                                 const CCValAssign &VA, const SDLoc &DL) {
EVT LocVT = VA.getLocVT();

switch (VA.getLocInfo()) {
default:
  llvm_unreachable("Unexpected CCValAssign::LocInfo")::llvm::llvm_unreachable_internal("Unexpected CCValAssign::LocInfo"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5052);
case CCValAssign::Full:
  break;
case CCValAssign::BCvt:
  if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
    Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
  else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
    Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
  else
    Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
  break;
}
return Val;
5065}

5067// The caller is responsible for loading the full value if the argument is
5068// passed with CCValAssign::Indirect.
5069static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
                              const CCValAssign &VA, const SDLoc &DL) {
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo &MFI = MF.getFrameInfo();
EVT LocVT = VA.getLocVT();
EVT ValVT = VA.getValVT();
EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
                               VA.getLocMemOffset(), /*Immutable=*/true);
SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
SDValue Val;

ISD::LoadExtType ExtType;
switch (VA.getLocInfo()) {
default:
  llvm_unreachable("Unexpected CCValAssign::LocInfo")::llvm::llvm_unreachable_internal("Unexpected CCValAssign::LocInfo"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5084);
case CCValAssign::Full:
case CCValAssign::Indirect:
case CCValAssign::BCvt:
  ExtType = ISD::NON_EXTLOAD;
  break;
}
Val = DAG.getExtLoad(
    ExtType, DL, LocVT, Chain, FIN,
    MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
return Val;
5095}

5097static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
                                     const CCValAssign &VA, const SDLoc &DL) {
assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&((VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::
f64 && "Unexpected VA") ? static_cast<void> (0)
 : __assert_fail ("VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 && \"Unexpected VA\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5100, __PRETTY_FUNCTION__))
       "Unexpected VA")((VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::
f64 && "Unexpected VA") ? static_cast<void> (0)
 : __assert_fail ("VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 && \"Unexpected VA\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5100, __PRETTY_FUNCTION__));
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo &MFI = MF.getFrameInfo();
MachineRegisterInfo &RegInfo = MF.getRegInfo();

if (VA.isMemLoc()) {
  // f64 is passed on the stack.
  int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
  return DAG.getLoad(MVT::f64, DL, Chain, FIN,
                     MachinePointerInfo::getFixedStack(MF, FI));
}

assert(VA.isRegLoc() && "Expected register VA assignment")((VA.isRegLoc() && "Expected register VA assignment")
 ? static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Expected register VA assignment\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5113, __PRETTY_FUNCTION__));

Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
SDValue Hi;
if (VA.getLocReg() == RISCV::X17) {
  // Second half of f64 is passed on the stack.
  int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
  Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
                   MachinePointerInfo::getFixedStack(MF, FI));
} else {
  // Second half of f64 is passed in another GPR.
  Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
  RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
  Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
}
return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
5132}

5134// FastCC has less than 1% performance improvement for some particular
5135// benchmark. But theoretically, it may has benenfit for some cases.
5136static bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
                          CCValAssign::LocInfo LocInfo,
                          ISD::ArgFlagsTy ArgFlags, CCState &State) {

if (LocVT == MVT::i32 || LocVT == MVT::i64) {
  // X5 and X6 might be used for save-restore libcall.
  static const MCPhysReg GPRList[] = {
      RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
      RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7,  RISCV::X28,
      RISCV::X29, RISCV::X30, RISCV::X31};
  if (unsigned Reg = State.AllocateReg(GPRList)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

if (LocVT == MVT::f16) {
  static const MCPhysReg FPR16List[] = {
      RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
      RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
      RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
      RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
  if (unsigned Reg = State.AllocateReg(FPR16List)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

if (LocVT == MVT::f32) {
  static const MCPhysReg FPR32List[] = {
      RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
      RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F,  RISCV::F1_F,
      RISCV::F2_F,  RISCV::F3_F,  RISCV::F4_F,  RISCV::F5_F,  RISCV::F6_F,
      RISCV::F7_F,  RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
  if (unsigned Reg = State.AllocateReg(FPR32List)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

if (LocVT == MVT::f64) {
  static const MCPhysReg FPR64List[] = {
      RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
      RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D,  RISCV::F1_D,
      RISCV::F2_D,  RISCV::F3_D,  RISCV::F4_D,  RISCV::F5_D,  RISCV::F6_D,
      RISCV::F7_D,  RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
  if (unsigned Reg = State.AllocateReg(FPR64List)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

if (LocVT == MVT::i32 || LocVT == MVT::f32) {
  unsigned Offset4 = State.AllocateStack(4, Align(4));
  State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
  return false;
}

if (LocVT == MVT::i64 || LocVT == MVT::f64) {
  unsigned Offset5 = State.AllocateStack(8, Align(8));
  State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
  return false;
}

return true; // CC didn't match.
5201}

5203static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                       CCValAssign::LocInfo LocInfo,
                       ISD::ArgFlagsTy ArgFlags, CCState &State) {

if (LocVT == MVT::i32 || LocVT == MVT::i64) {
  // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
  //                        s1    s2  s3  s4  s5  s6  s7  s8  s9  s10 s11
  static const MCPhysReg GPRList[] = {
      RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
      RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
  if (unsigned Reg = State.AllocateReg(GPRList)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

if (LocVT == MVT::f32) {
  // Pass in STG registers: F1, ..., F6
  //                        fs0 ... fs5
  static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
                                        RISCV::F18_F, RISCV::F19_F,
                                        RISCV::F20_F, RISCV::F21_F};
  if (unsigned Reg = State.AllocateReg(FPR32List)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

if (LocVT == MVT::f64) {
  // Pass in STG registers: D1, ..., D6
  //                        fs6 ... fs11
  static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
                                        RISCV::F24_D, RISCV::F25_D,
                                        RISCV::F26_D, RISCV::F27_D};
  if (unsigned Reg = State.AllocateReg(FPR64List)) {
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
    return false;
  }
}

report_fatal_error("No registers left in GHC calling convention");
return true;
5245}

5247// Transform physical registers into virtual registers.
5248SDValue RISCVTargetLowering::LowerFormalArguments(
  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {

MachineFunction &MF = DAG.getMachineFunction();

switch (CallConv) {
default:
  report_fatal_error("Unsupported calling convention");
case CallingConv::C:
case CallingConv::Fast:
  break;
case CallingConv::GHC:
  if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
      !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
    report_fatal_error(
      "GHC calling convention requires the F and D instruction set extensions");
}

const Function &Func = MF.getFunction();
if (Func.hasFnAttribute("interrupt")) {
  if (!Func.arg_empty())
    report_fatal_error(
      "Functions with the interrupt attribute cannot have arguments!");

  StringRef Kind =
    MF.getFunction().getFnAttribute("interrupt").getValueAsString();

  if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
    report_fatal_error(
      "Function interrupt attribute argument not supported!");
}

EVT PtrVT = getPointerTy(DAG.getDataLayout());
MVT XLenVT = Subtarget.getXLenVT();
unsigned XLenInBytes = Subtarget.getXLen() / 8;
// Used with vargs to acumulate store chains.
std::vector<SDValue> OutChains;

// Assign locations to all of the incoming arguments.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());

if (CallConv == CallingConv::Fast)
  CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_FastCC);
else if (CallConv == CallingConv::GHC)
  CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
else
  analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);

for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
  CCValAssign &VA = ArgLocs[i];
  SDValue ArgValue;
  // Passing f64 on RV32D with a soft float ABI must be handled as a special
  // case.
  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
    ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
  else if (VA.isRegLoc())
    ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
  else
    ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);

  if (VA.getLocInfo() == CCValAssign::Indirect) {
    // If the original argument was split and passed by reference (e.g. i128
    // on RV32), we need to load all parts of it here (using the same
    // address).
    InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
                                 MachinePointerInfo()));
    unsigned ArgIndex = Ins[i].OrigArgIndex;
    assert(Ins[i].PartOffset == 0)((Ins[i].PartOffset == 0) ? static_cast<void> (0) : __assert_fail
 ("Ins[i].PartOffset == 0", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5318, __PRETTY_FUNCTION__));
    while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
      CCValAssign &PartVA = ArgLocs[i + 1];
      unsigned PartOffset = Ins[i + 1].PartOffset;
      SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
                                    DAG.getIntPtrConstant(PartOffset, DL));
      InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
                                   MachinePointerInfo()));
      ++i;
    }
    continue;
  }
  InVals.push_back(ArgValue);
}

if (IsVarArg) {
  ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
  unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
  const TargetRegisterClass *RC = &RISCV::GPRRegClass;
  MachineFrameInfo &MFI = MF.getFrameInfo();
  MachineRegisterInfo &RegInfo = MF.getRegInfo();
  RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();

  // Offset of the first variable argument from stack pointer, and size of
  // the vararg save area. For now, the varargs save area is either zero or
  // large enough to hold a0-a7.
  int VaArgOffset, VarArgsSaveSize;

  // If all registers are allocated, then all varargs must be passed on the
  // stack and we don't need to save any argregs.
  if (ArgRegs.size() == Idx) {
    VaArgOffset = CCInfo.getNextStackOffset();
    VarArgsSaveSize = 0;
  } else {
    VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
    VaArgOffset = -VarArgsSaveSize;
  }

  // Record the frame index of the first variable argument
  // which is a value necessary to VASTART.
  int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
  RVFI->setVarArgsFrameIndex(FI);

  // If saving an odd number of registers then create an extra stack slot to
  // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
  // offsets to even-numbered registered remain 2*XLEN-aligned.
  if (Idx % 2) {
    MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
    VarArgsSaveSize += XLenInBytes;
  }

  // Copy the integer registers that may have been used for passing varargs
  // to the vararg save area.
  for (unsigned I = Idx; I < ArgRegs.size();
       ++I, VaArgOffset += XLenInBytes) {
    const Register Reg = RegInfo.createVirtualRegister(RC);
    RegInfo.addLiveIn(ArgRegs[I], Reg);
    SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
    FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
    SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
    SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
                                 MachinePointerInfo::getFixedStack(MF, FI));
    cast<StoreSDNode>(Store.getNode())
        ->getMemOperand()
        ->setValue((Value *)nullptr);
    OutChains.push_back(Store);
  }
  RVFI->setVarArgsSaveSize(VarArgsSaveSize);
}

// All stores are grouped in one node to allow the matching between
// the size of Ins and InVals. This only happens for vararg functions.
if (!OutChains.empty()) {
  OutChains.push_back(Chain);
  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
}

return Chain;
5396}

5398/// isEligibleForTailCallOptimization - Check whether the call is eligible
5399/// for tail call optimization.
5400/// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
5401bool RISCVTargetLowering::isEligibleForTailCallOptimization(
  CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
  const SmallVector<CCValAssign, 16> &ArgLocs) const {

auto &Callee = CLI.Callee;
auto CalleeCC = CLI.CallConv;
auto &Outs = CLI.Outs;
auto &Caller = MF.getFunction();
auto CallerCC = Caller.getCallingConv();

// Exception-handling functions need a special set of instructions to
// indicate a return to the hardware. Tail-calling another function would
// probably break this.
// TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
// should be expanded as new function attributes are introduced.
if (Caller.hasFnAttribute("interrupt"))
  return false;

// Do not tail call opt if the stack is used to pass parameters.
if (CCInfo.getNextStackOffset() != 0)
  return false;

// Do not tail call opt if any parameters need to be passed indirectly.
// Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
// passed indirectly. So the address of the value will be passed in a
// register, or if not available, then the address is put on the stack. In
// order to pass indirectly, space on the stack often needs to be allocated
// in order to store the value. In this case the CCInfo.getNextStackOffset()
// != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
// are passed CCValAssign::Indirect.
for (auto &VA : ArgLocs)
  if (VA.getLocInfo() == CCValAssign::Indirect)
    return false;

// Do not tail call opt if either caller or callee uses struct return
// semantics.
auto IsCallerStructRet = Caller.hasStructRetAttr();
auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
if (IsCallerStructRet || IsCalleeStructRet)
  return false;

// Externally-defined functions with weak linkage should not be
// tail-called. The behaviour of branch instructions in this situation (as
// used for tail calls) is implementation-defined, so we cannot rely on the
// linker replacing the tail call with a return.
if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
  const GlobalValue *GV = G->getGlobal();
  if (GV->hasExternalWeakLinkage())
    return false;
}

// The callee has to preserve all registers the caller needs to preserve.
const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
if (CalleeCC != CallerCC) {
  const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
  if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
    return false;
}

// Byval parameters hand the function a pointer directly into the stack area
// we want to reuse during a tail call. Working around this *is* possible
// but less efficient and uglier in LowerCall.
for (auto &Arg : Outs)
  if (Arg.Flags.isByVal())
    return false;

return true;
5469}

5471// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
5472// and output parameter nodes.
5473SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
                                     SmallVectorImpl<SDValue> &InVals) const {
SelectionDAG &DAG = CLI.DAG;
SDLoc &DL = CLI.DL;
SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
bool &IsTailCall = CLI.IsTailCall;
CallingConv::ID CallConv = CLI.CallConv;
bool IsVarArg = CLI.IsVarArg;
EVT PtrVT = getPointerTy(DAG.getDataLayout());
MVT XLenVT = Subtarget.getXLenVT();

MachineFunction &MF = DAG.getMachineFunction();

// Analyze the operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());

if (CallConv == CallingConv::Fast)
  ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_FastCC);
else if (CallConv == CallingConv::GHC)
  ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
else
  analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI);

// Check if it's really possible to do a tail call.
if (IsTailCall)
  IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);

if (IsTailCall)
  ++NumTailCalls;
else if (CLI.CB && CLI.CB->isMustTailCall())
  report_fatal_error("failed to perform tail call elimination on a call "
                     "site marked musttail");

// Get a count of how many bytes are to be pushed on the stack.
unsigned NumBytes = ArgCCInfo.getNextStackOffset();

// Create local copies for byval args
SmallVector<SDValue, 8> ByValArgs;
for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
  ISD::ArgFlagsTy Flags = Outs[i].Flags;
  if (!Flags.isByVal())
    continue;

  SDValue Arg = OutVals[i];
  unsigned Size = Flags.getByValSize();
  Align Alignment = Flags.getNonZeroByValAlign();

  int FI =
      MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
  SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
  SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);

  Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
                        /*IsVolatile=*/false,
                        /*AlwaysInline=*/false, IsTailCall,
                        MachinePointerInfo(), MachinePointerInfo());
  ByValArgs.push_back(FIPtr);
}

if (!IsTailCall)
  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);

// Copy argument values to their designated locations.
SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
SmallVector<SDValue, 8> MemOpChains;
SDValue StackPtr;
for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
  CCValAssign &VA = ArgLocs[i];
  SDValue ArgValue = OutVals[i];
  ISD::ArgFlagsTy Flags = Outs[i].Flags;

  // Handle passing f64 on RV32D with a soft float ABI as a special case.
  bool IsF64OnRV32DSoftABI =
      VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
  if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
    SDValue SplitF64 = DAG.getNode(
        RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
    SDValue Lo = SplitF64.getValue(0);
    SDValue Hi = SplitF64.getValue(1);

    Register RegLo = VA.getLocReg();
    RegsToPass.push_back(std::make_pair(RegLo, Lo));

    if (RegLo == RISCV::X17) {
      // Second half of f64 is passed on the stack.
      // Work out the address of the stack slot.
      if (!StackPtr.getNode())
        StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
      // Emit the store.
      MemOpChains.push_back(
          DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
    } else {
      // Second half of f64 is passed in another GPR.
      assert(RegLo < RISCV::X31 && "Invalid register pair")((RegLo < RISCV::X31 && "Invalid register pair") ?
 static_cast<void> (0) : __assert_fail ("RegLo < RISCV::X31 && \"Invalid register pair\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5571, __PRETTY_FUNCTION__));
      Register RegHigh = RegLo + 1;
      RegsToPass.push_back(std::make_pair(RegHigh, Hi));
    }
    continue;
  }

  // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
  // as any other MemLoc.

  // Promote the value if needed.
  // For now, only handle fully promoted and indirect arguments.
  if (VA.getLocInfo() == CCValAssign::Indirect) {
    // Store the argument in a stack slot and pass its address.
    SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
    int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
    MemOpChains.push_back(
        DAG.getStore(Chain, DL, ArgValue, SpillSlot,
                     MachinePointerInfo::getFixedStack(MF, FI)));
    // If the original argument was split (e.g. i128), we need
    // to store all parts of it here (and pass just one address).
    unsigned ArgIndex = Outs[i].OrigArgIndex;
    assert(Outs[i].PartOffset == 0)((Outs[i].PartOffset == 0) ? static_cast<void> (0) : __assert_fail
 ("Outs[i].PartOffset == 0", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5593, __PRETTY_FUNCTION__));
    while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
      SDValue PartValue = OutVals[i + 1];
      unsigned PartOffset = Outs[i + 1].PartOffset;
      SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
                                    DAG.getIntPtrConstant(PartOffset, DL));
      MemOpChains.push_back(
          DAG.getStore(Chain, DL, PartValue, Address,
                       MachinePointerInfo::getFixedStack(MF, FI)));
      ++i;
    }
    ArgValue = SpillSlot;
  } else {
    ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
  }

  // Use local copy if it is a byval arg.
  if (Flags.isByVal())
    ArgValue = ByValArgs[j++];

  if (VA.isRegLoc()) {
    // Queue up the argument copies and emit them at the end.
    RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
  } else {
    assert(VA.isMemLoc() && "Argument not register or memory")((VA.isMemLoc() && "Argument not register or memory")
 ? static_cast<void> (0) : __assert_fail ("VA.isMemLoc() && \"Argument not register or memory\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5617, __PRETTY_FUNCTION__));
    assert(!IsTailCall && "Tail call not allowed if stack is used "((!IsTailCall && "Tail call not allowed if stack is used "
 "for passing parameters") ? static_cast<void> (0) : __assert_fail
 ("!IsTailCall && \"Tail call not allowed if stack is used \" \"for passing parameters\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5619, __PRETTY_FUNCTION__))
                          "for passing parameters")((!IsTailCall && "Tail call not allowed if stack is used "
 "for passing parameters") ? static_cast<void> (0) : __assert_fail
 ("!IsTailCall && \"Tail call not allowed if stack is used \" \"for passing parameters\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5619, __PRETTY_FUNCTION__));

    // Work out the address of the stack slot.
    if (!StackPtr.getNode())
      StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
    SDValue Address =
        DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
                    DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));

    // Emit the store.
    MemOpChains.push_back(
        DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
  }
}

// Join the stores, which are independent of one another.
if (!MemOpChains.empty())
  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);

SDValue Glue;

// Build a sequence of copy-to-reg nodes, chained and glued together.
for (auto &Reg : RegsToPass) {
  Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
  Glue = Chain.getValue(1);
}

// Validate that none of the argument registers have been marked as
// reserved, if so report an error. Do the same for the return address if this
// is not a tailcall.
validateCCReservedRegs(RegsToPass, MF);
if (!IsTailCall &&
    MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
  MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
      MF.getFunction(),
      "Return address register required, but has been reserved."});

// If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
// TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
// split it and then direct call can be matched by PseudoCALL.
if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
  const GlobalValue *GV = S->getGlobal();

  unsigned OpFlags = RISCVII::MO_CALL;
  if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
    OpFlags = RISCVII::MO_PLT;

  Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
} else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
  unsigned OpFlags = RISCVII::MO_CALL;

  if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
                                               nullptr))
    OpFlags = RISCVII::MO_PLT;

  Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
}

// The first call operand is the chain and the second is the target address.
SmallVector<SDValue, 8> Ops;
Ops.push_back(Chain);
Ops.push_back(Callee);

// Add argument registers to the end of the list so that they are
// known live into the call.
for (auto &Reg : RegsToPass)
  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));

if (!IsTailCall) {
  // Add a register mask operand representing the call-preserved registers.
  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
  assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention"
) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5691, __PRETTY_FUNCTION__));
  Ops.push_back(DAG.getRegisterMask(Mask));
}

// Glue the call to the argument copies, if any.
if (Glue.getNode())
  Ops.push_back(Glue);

// Emit the call.
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);

if (IsTailCall) {
  MF.getFrameInfo().setHasTailCall();
  return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
}

Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
Glue = Chain.getValue(1);

// Mark the end of the call, which is glued to the call itself.
Chain = DAG.getCALLSEQ_END(Chain,
                           DAG.getConstant(NumBytes, DL, PtrVT, true),
                           DAG.getConstant(0, DL, PtrVT, true),
                           Glue, DL);
Glue = Chain.getValue(1);

// Assign locations to each value returned by this call.
SmallVector<CCValAssign, 16> RVLocs;
CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);

// Copy all of the result registers out of their specified physreg.
for (auto &VA : RVLocs) {
  // Copy the value out
  SDValue RetValue =
      DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
  // Glue the RetValue to the end of the call sequence
  Chain = RetValue.getValue(1);
  Glue = RetValue.getValue(2);

  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
    assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment")((VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment"
) ? static_cast<void> (0) : __assert_fail ("VA.getLocReg() == ArgGPRs[0] && \"Unexpected reg assignment\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5733, __PRETTY_FUNCTION__));
    SDValue RetValue2 =
        DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
    Chain = RetValue2.getValue(1);
    Glue = RetValue2.getValue(2);
    RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
                           RetValue2);
  }

  RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);

  InVals.push_back(RetValue);
}

return Chain;
5748}

5750bool RISCVTargetLowering::CanLowerReturn(
  CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
  const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
SmallVector<CCValAssign, 16> RVLocs;
CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);

Optional<unsigned> FirstMaskArgument;
if (Subtarget.hasStdExtV())
  FirstMaskArgument = preAssignMask(Outs);

for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
  MVT VT = Outs[i].VT;
  ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
  if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
               ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
               *this, FirstMaskArgument))
    return false;
}
return true;
5770}

5772SDValue
5773RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                               bool IsVarArg,
                               const SmallVectorImpl<ISD::OutputArg> &Outs,
                               const SmallVectorImpl<SDValue> &OutVals,
                               const SDLoc &DL, SelectionDAG &DAG) const {
const MachineFunction &MF = DAG.getMachineFunction();
const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();

// Stores the assignment of the return value to a location.
SmallVector<CCValAssign, 16> RVLocs;

// Info about the registers and stack slot.
CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
               *DAG.getContext());

analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
                  nullptr);

if (CallConv == CallingConv::GHC && !RVLocs.empty())
  report_fatal_error("GHC functions return void only");

SDValue Glue;
SmallVector<SDValue, 4> RetOps(1, Chain);

// Copy the result values into the output registers.
for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
  SDValue Val = OutVals[i];
  CCValAssign &VA = RVLocs[i];
  assert(VA.isRegLoc() && "Can only return in registers!")((VA.isRegLoc() && "Can only return in registers!") ?
 static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5801, __PRETTY_FUNCTION__));

  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
    // Handle returning f64 on RV32D with a soft float ABI.
    assert(VA.isRegLoc() && "Expected return via registers")((VA.isRegLoc() && "Expected return via registers") ?
 static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Expected return via registers\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5805, __PRETTY_FUNCTION__));
    SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
                                   DAG.getVTList(MVT::i32, MVT::i32), Val);
    SDValue Lo = SplitF64.getValue(0);
    SDValue Hi = SplitF64.getValue(1);
    Register RegLo = VA.getLocReg();
    assert(RegLo < RISCV::X31 && "Invalid register pair")((RegLo < RISCV::X31 && "Invalid register pair") ?
 static_cast<void> (0) : __assert_fail ("RegLo < RISCV::X31 && \"Invalid register pair\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 5811, __PRETTY_FUNCTION__));
    Register RegHi = RegLo + 1;

    if (STI.isRegisterReservedByUser(RegLo) ||
        STI.isRegisterReservedByUser(RegHi))
      MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
          MF.getFunction(),
          "Return value register required, but has been reserved."});

    Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
    Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
  } else {
    // Handle a 'normal' return.
    Val = convertValVTToLocVT(DAG, Val, VA, DL);
    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);

    if (STI.isRegisterReservedByUser(VA.getLocReg()))
      MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
          MF.getFunction(),
          "Return value register required, but has been reserved."});

    // Guarantee that all emitted copies are stuck together.
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
  }
}

RetOps[0] = Chain; // Update chain.

// Add the glue node if we have it.
if (Glue.getNode()) {
  RetOps.push_back(Glue);
}

// Interrupt service routines use different return instructions.
const Function &Func = DAG.getMachineFunction().getFunction();
if (Func.hasFnAttribute("interrupt")) {
  if (!Func.getReturnType()->isVoidTy())
    report_fatal_error(
        "Functions with the interrupt attribute must have void return type!");

  MachineFunction &MF = DAG.getMachineFunction();
  StringRef Kind =
    MF.getFunction().getFnAttribute("interrupt").getValueAsString();

  unsigned RetOpc;
  if (Kind == "user")
    RetOpc = RISCVISD::URET_FLAG;
  else if (Kind == "supervisor")
    RetOpc = RISCVISD::SRET_FLAG;
  else
    RetOpc = RISCVISD::MRET_FLAG;

  return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
}

return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
5872}

5874void RISCVTargetLowering::validateCCReservedRegs(
  const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
  MachineFunction &MF) const {
const Function &F = MF.getFunction();
const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();

if (llvm::any_of(Regs, [&STI](auto Reg) {
      return STI.isRegisterReservedByUser(Reg.first);
    }))
  F.getContext().diagnose(DiagnosticInfoUnsupported{
      F, "Argument register required, but has been reserved."});
5885}

5887bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
return CI->isTailCall();
5889}

5891const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
5892#define NODE_NAME_CASE(NODE)                                                   \
case RISCVISD::NODE:                                                         \
  return "RISCVISD::" #NODE;
// clang-format off
switch ((RISCVISD::NodeType)Opcode) {
case RISCVISD::FIRST_NUMBER:
  break;
NODE_NAME_CASE(RET_FLAG)
NODE_NAME_CASE(URET_FLAG)
NODE_NAME_CASE(SRET_FLAG)
NODE_NAME_CASE(MRET_FLAG)
NODE_NAME_CASE(CALL)
NODE_NAME_CASE(SELECT_CC)
NODE_NAME_CASE(BuildPairF64)
NODE_NAME_CASE(SplitF64)
NODE_NAME_CASE(TAIL)
NODE_NAME_CASE(SLLW)
NODE_NAME_CASE(SRAW)
NODE_NAME_CASE(SRLW)
NODE_NAME_CASE(DIVW)
NODE_NAME_CASE(DIVUW)
NODE_NAME_CASE(REMUW)
NODE_NAME_CASE(ROLW)
NODE_NAME_CASE(RORW)
NODE_NAME_CASE(FSLW)
NODE_NAME_CASE(FSRW)
NODE_NAME_CASE(FSL)
NODE_NAME_CASE(FSR)
NODE_NAME_CASE(FMV_H_X)
NODE_NAME_CASE(FMV_X_ANYEXTH)
NODE_NAME_CASE(FMV_W_X_RV64)
NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
NODE_NAME_CASE(READ_CYCLE_WIDE)
NODE_NAME_CASE(GREVI)
NODE_NAME_CASE(GREVIW)
NODE_NAME_CASE(GORCI)
NODE_NAME_CASE(GORCIW)
NODE_NAME_CASE(SHFLI)
NODE_NAME_CASE(VMV_V_X_VL)
NODE_NAME_CASE(VFMV_V_F_VL)
NODE_NAME_CASE(VMV_X_S)
NODE_NAME_CASE(VMV_S_XF_VL)
NODE_NAME_CASE(SPLAT_VECTOR_I64)
NODE_NAME_CASE(READ_VLENB)
NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
NODE_NAME_CASE(VLEFF)
NODE_NAME_CASE(VLEFF_MASK)
NODE_NAME_CASE(VSLIDEUP_VL)
NODE_NAME_CASE(VSLIDEDOWN_VL)
NODE_NAME_CASE(VID_VL)
NODE_NAME_CASE(VFNCVT_ROD_VL)
NODE_NAME_CASE(VECREDUCE_ADD_VL)
NODE_NAME_CASE(VECREDUCE_UMAX_VL)
NODE_NAME_CASE(VECREDUCE_SMAX_VL)
NODE_NAME_CASE(VECREDUCE_UMIN_VL)
NODE_NAME_CASE(VECREDUCE_SMIN_VL)
NODE_NAME_CASE(VECREDUCE_AND_VL)
NODE_NAME_CASE(VECREDUCE_OR_VL)
NODE_NAME_CASE(VECREDUCE_XOR_VL)
NODE_NAME_CASE(VECREDUCE_FADD_VL)
NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
NODE_NAME_CASE(ADD_VL)
NODE_NAME_CASE(AND_VL)
NODE_NAME_CASE(MUL_VL)
NODE_NAME_CASE(OR_VL)
NODE_NAME_CASE(SDIV_VL)
NODE_NAME_CASE(SHL_VL)
NODE_NAME_CASE(SREM_VL)
NODE_NAME_CASE(SRA_VL)
NODE_NAME_CASE(SRL_VL)
NODE_NAME_CASE(SUB_VL)
NODE_NAME_CASE(UDIV_VL)
NODE_NAME_CASE(UREM_VL)
NODE_NAME_CASE(XOR_VL)
NODE_NAME_CASE(FADD_VL)
NODE_NAME_CASE(FSUB_VL)
NODE_NAME_CASE(FMUL_VL)
NODE_NAME_CASE(FDIV_VL)
NODE_NAME_CASE(FNEG_VL)
NODE_NAME_CASE(FABS_VL)
NODE_NAME_CASE(FSQRT_VL)
NODE_NAME_CASE(FMA_VL)
NODE_NAME_CASE(SMIN_VL)
NODE_NAME_CASE(SMAX_VL)
NODE_NAME_CASE(UMIN_VL)
NODE_NAME_CASE(UMAX_VL)
NODE_NAME_CASE(MULHS_VL)
NODE_NAME_CASE(MULHU_VL)
NODE_NAME_CASE(FP_TO_SINT_VL)
NODE_NAME_CASE(FP_TO_UINT_VL)
NODE_NAME_CASE(SINT_TO_FP_VL)
NODE_NAME_CASE(UINT_TO_FP_VL)
NODE_NAME_CASE(FP_EXTEND_VL)
NODE_NAME_CASE(FP_ROUND_VL)
NODE_NAME_CASE(SETCC_VL)
NODE_NAME_CASE(VSELECT_VL)
NODE_NAME_CASE(VMAND_VL)
NODE_NAME_CASE(VMOR_VL)
NODE_NAME_CASE(VMXOR_VL)
NODE_NAME_CASE(VMCLR_VL)
NODE_NAME_CASE(VMSET_VL)
NODE_NAME_CASE(VRGATHER_VX_VL)
NODE_NAME_CASE(VRGATHER_VV_VL)
NODE_NAME_CASE(VRGATHEREI16_VV_VL)
NODE_NAME_CASE(VSEXT_VL)
NODE_NAME_CASE(VZEXT_VL)
NODE_NAME_CASE(VLE_VL)
NODE_NAME_CASE(VSE_VL)
}
// clang-format on
return nullptr;
6003#undef NODE_NAME_CASE
6004}

6006/// getConstraintType - Given a constraint letter, return the type of
6007/// constraint it is for this target.
6008RISCVTargetLowering::ConstraintType
6009RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
if (Constraint.size() == 1) {
  switch (Constraint[0]) {
  default:
    break;
  case 'f':
    return C_RegisterClass;
  case 'I':
  case 'J':
  case 'K':
    return C_Immediate;
  case 'A':
    return C_Memory;
  }
}
return TargetLowering::getConstraintType(Constraint);
6025}

6027std::pair<unsigned, const TargetRegisterClass *>
6028RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
                                                StringRef Constraint,
                                                MVT VT) const {
// First, see if this is a constraint that directly corresponds to a
// RISCV register class.
if (Constraint.size() == 1) {
  switch (Constraint[0]) {
  case 'r':
    return std::make_pair(0U, &RISCV::GPRRegClass);
  case 'f':
    if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
      return std::make_pair(0U, &RISCV::FPR16RegClass);
    if (Subtarget.hasStdExtF() && VT == MVT::f32)
      return std::make_pair(0U, &RISCV::FPR32RegClass);
    if (Subtarget.hasStdExtD() && VT == MVT::f64)
      return std::make_pair(0U, &RISCV::FPR64RegClass);
    break;
  default:
    break;
  }
}

// Clang will correctly decode the usage of register name aliases into their
// official names. However, other frontends like `rustc` do not. This allows
// users of these frontends to use the ABI names for registers in LLVM-style
// register constraints.
unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
                             .Case("{zero}", RISCV::X0)
                             .Case("{ra}", RISCV::X1)
                             .Case("{sp}", RISCV::X2)
                             .Case("{gp}", RISCV::X3)
                             .Case("{tp}", RISCV::X4)
                             .Case("{t0}", RISCV::X5)
                             .Case("{t1}", RISCV::X6)
                             .Case("{t2}", RISCV::X7)
                             .Cases("{s0}", "{fp}", RISCV::X8)
                             .Case("{s1}", RISCV::X9)
                             .Case("{a0}", RISCV::X10)
                             .Case("{a1}", RISCV::X11)
                             .Case("{a2}", RISCV::X12)
                             .Case("{a3}", RISCV::X13)
                             .Case("{a4}", RISCV::X14)
                             .Case("{a5}", RISCV::X15)
                             .Case("{a6}", RISCV::X16)
                             .Case("{a7}", RISCV::X17)
                             .Case("{s2}", RISCV::X18)
                             .Case("{s3}", RISCV::X19)
                             .Case("{s4}", RISCV::X20)
                             .Case("{s5}", RISCV::X21)
                             .Case("{s6}", RISCV::X22)
                             .Case("{s7}", RISCV::X23)
                             .Case("{s8}", RISCV::X24)
                             .Case("{s9}", RISCV::X25)
                             .Case("{s10}", RISCV::X26)
                             .Case("{s11}", RISCV::X27)
                             .Case("{t3}", RISCV::X28)
                             .Case("{t4}", RISCV::X29)
                             .Case("{t5}", RISCV::X30)
                             .Case("{t6}", RISCV::X31)
                             .Default(RISCV::NoRegister);
if (XRegFromAlias != RISCV::NoRegister)
  return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);

// Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
// TableGen record rather than the AsmName to choose registers for InlineAsm
// constraints, plus we want to match those names to the widest floating point
// register type available, manually select floating point registers here.
//
// The second case is the ABI name of the register, so that frontends can also
// use the ABI names in register constraint lists.
if (Subtarget.hasStdExtF()) {
  unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
                      .Cases("{f0}", "{ft0}", RISCV::F0_F)
                      .Cases("{f1}", "{ft1}", RISCV::F1_F)
                      .Cases("{f2}", "{ft2}", RISCV::F2_F)
                      .Cases("{f3}", "{ft3}", RISCV::F3_F)
                      .Cases("{f4}", "{ft4}", RISCV::F4_F)
                      .Cases("{f5}", "{ft5}", RISCV::F5_F)
                      .Cases("{f6}", "{ft6}", RISCV::F6_F)
                      .Cases("{f7}", "{ft7}", RISCV::F7_F)
                      .Cases("{f8}", "{fs0}", RISCV::F8_F)
                      .Cases("{f9}", "{fs1}", RISCV::F9_F)
                      .Cases("{f10}", "{fa0}", RISCV::F10_F)
                      .Cases("{f11}", "{fa1}", RISCV::F11_F)
                      .Cases("{f12}", "{fa2}", RISCV::F12_F)
                      .Cases("{f13}", "{fa3}", RISCV::F13_F)
                      .Cases("{f14}", "{fa4}", RISCV::F14_F)
                      .Cases("{f15}", "{fa5}", RISCV::F15_F)
                      .Cases("{f16}", "{fa6}", RISCV::F16_F)
                      .Cases("{f17}", "{fa7}", RISCV::F17_F)
                      .Cases("{f18}", "{fs2}", RISCV::F18_F)
                      .Cases("{f19}", "{fs3}", RISCV::F19_F)
                      .Cases("{f20}", "{fs4}", RISCV::F20_F)
                      .Cases("{f21}", "{fs5}", RISCV::F21_F)
                      .Cases("{f22}", "{fs6}", RISCV::F22_F)
                      .Cases("{f23}", "{fs7}", RISCV::F23_F)
                      .Cases("{f24}", "{fs8}", RISCV::F24_F)
                      .Cases("{f25}", "{fs9}", RISCV::F25_F)
                      .Cases("{f26}", "{fs10}", RISCV::F26_F)
                      .Cases("{f27}", "{fs11}", RISCV::F27_F)
                      .Cases("{f28}", "{ft8}", RISCV::F28_F)
                      .Cases("{f29}", "{ft9}", RISCV::F29_F)
                      .Cases("{f30}", "{ft10}", RISCV::F30_F)
                      .Cases("{f31}", "{ft11}", RISCV::F31_F)
                      .Default(RISCV::NoRegister);
  if (FReg != RISCV::NoRegister) {
    assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg")((RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
 "Unknown fp-reg") ? static_cast<void> (0) : __assert_fail
 ("RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && \"Unknown fp-reg\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 6134, __PRETTY_FUNCTION__));
    if (Subtarget.hasStdExtD()) {
      unsigned RegNo = FReg - RISCV::F0_F;
      unsigned DReg = RISCV::F0_D + RegNo;
      return std::make_pair(DReg, &RISCV::FPR64RegClass);
    }
    return std::make_pair(FReg, &RISCV::FPR32RegClass);
  }
}

return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
6145}

6147unsigned
6148RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
// Currently only support length 1 constraints.
if (ConstraintCode.size() == 1) {
  switch (ConstraintCode[0]) {
  case 'A':
    return InlineAsm::Constraint_A;
  default:
    break;
  }
}

return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
6160}

6162void RISCVTargetLowering::LowerAsmOperandForConstraint(
  SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
  SelectionDAG &DAG) const {
// Currently only support length 1 constraints.
if (Constraint.length() == 1) {
  switch (Constraint[0]) {
  case 'I':
    // Validate & create a 12-bit signed immediate operand.
    if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
      uint64_t CVal = C->getSExtValue();
      if (isInt<12>(CVal))
        Ops.push_back(
            DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
    }
    return;
  case 'J':
    // Validate & create an integer zero operand.
    if (auto *C = dyn_cast<ConstantSDNode>(Op))
      if (C->getZExtValue() == 0)
        Ops.push_back(
            DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
    return;
  case 'K':
    // Validate & create a 5-bit unsigned immediate operand.
    if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
      uint64_t CVal = C->getZExtValue();
      if (isUInt<5>(CVal))
        Ops.push_back(
            DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
    }
    return;
  default:
    break;
  }
}
TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
6198}

6200Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
                                                 Instruction *Inst,
                                                 AtomicOrdering Ord) const {
if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
  return Builder.CreateFence(Ord);
if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
  return Builder.CreateFence(AtomicOrdering::Release);
return nullptr;
6208}

6210Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
                                                  Instruction *Inst,
                                                  AtomicOrdering Ord) const {
if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
  return Builder.CreateFence(AtomicOrdering::Acquire);
return nullptr;
6216}

6218TargetLowering::AtomicExpansionKind
6219RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
// point operations can't be used in an lr/sc sequence without breaking the
// forward-progress guarantee.
if (AI->isFloatingPointOperation())
  return AtomicExpansionKind::CmpXChg;

unsigned Size = AI->getType()->getPrimitiveSizeInBits();
if (Size == 8 || Size == 16)
  return AtomicExpansionKind::MaskedIntrinsic;
return AtomicExpansionKind::None;
6230}

6232static Intrinsic::ID
6233getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
if (XLen == 32) {
  switch (BinOp) {
  default:
    llvm_unreachable("Unexpected AtomicRMW BinOp")::llvm::llvm_unreachable_internal("Unexpected AtomicRMW BinOp"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 6237);
  case AtomicRMWInst::Xchg:
    return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
  case AtomicRMWInst::Add:
    return Intrinsic::riscv_masked_atomicrmw_add_i32;
  case AtomicRMWInst::Sub:
    return Intrinsic::riscv_masked_atomicrmw_sub_i32;
  case AtomicRMWInst::Nand:
    return Intrinsic::riscv_masked_atomicrmw_nand_i32;
  case AtomicRMWInst::Max:
    return Intrinsic::riscv_masked_atomicrmw_max_i32;
  case AtomicRMWInst::Min:
    return Intrinsic::riscv_masked_atomicrmw_min_i32;
  case AtomicRMWInst::UMax:
    return Intrinsic::riscv_masked_atomicrmw_umax_i32;
  case AtomicRMWInst::UMin:
    return Intrinsic::riscv_masked_atomicrmw_umin_i32;
  }
}

if (XLen == 64) {
  switch (BinOp) {
  default:
    llvm_unreachable("Unexpected AtomicRMW BinOp")::llvm::llvm_unreachable_internal("Unexpected AtomicRMW BinOp"
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 6260);
  case AtomicRMWInst::Xchg:
    return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
  case AtomicRMWInst::Add:
    return Intrinsic::riscv_masked_atomicrmw_add_i64;
  case AtomicRMWInst::Sub:
    return Intrinsic::riscv_masked_atomicrmw_sub_i64;
  case AtomicRMWInst::Nand:
    return Intrinsic::riscv_masked_atomicrmw_nand_i64;
  case AtomicRMWInst::Max:
    return Intrinsic::riscv_masked_atomicrmw_max_i64;
  case AtomicRMWInst::Min:
    return Intrinsic::riscv_masked_atomicrmw_min_i64;
  case AtomicRMWInst::UMax:
    return Intrinsic::riscv_masked_atomicrmw_umax_i64;
  case AtomicRMWInst::UMin:
    return Intrinsic::riscv_masked_atomicrmw_umin_i64;
  }
}

llvm_unreachable("Unexpected XLen\n")::llvm::llvm_unreachable_internal("Unexpected XLen\n", "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 6280);
6281}

6283Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
  IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
  Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
unsigned XLen = Subtarget.getXLen();
Value *Ordering =
    Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
Type *Tys[] = {AlignedAddr->getType()};
Function *LrwOpScwLoop = Intrinsic::getDeclaration(
    AI->getModule(),
    getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);

if (XLen == 64) {
  Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
  ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
}

Value *Result;

// Must pass the shift amount needed to sign extend the loaded value prior
// to performing a signed comparison for min/max. ShiftAmt is the number of
// bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
// is the number of bits to left+right shift the value in order to
// sign-extend.
if (AI->getOperation() == AtomicRMWInst::Min ||
    AI->getOperation() == AtomicRMWInst::Max) {
  const DataLayout &DL = AI->getModule()->getDataLayout();
  unsigned ValWidth =
      DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
  Value *SextShamt =
      Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
  Result = Builder.CreateCall(LrwOpScwLoop,
                              {AlignedAddr, Incr, Mask, SextShamt, Ordering});
} else {
  Result =
      Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
}

if (XLen == 64)
  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
return Result;
6324}

6326TargetLowering::AtomicExpansionKind
6327RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
  AtomicCmpXchgInst *CI) const {
unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
if (Size == 8 || Size == 16)
  return AtomicExpansionKind::MaskedIntrinsic;
return AtomicExpansionKind::None;
6333}

6335Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
  IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
  Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
unsigned XLen = Subtarget.getXLen();
Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
if (XLen == 64) {
  CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
  NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
  CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
}
Type *Tys[] = {AlignedAddr->getType()};
Function *MaskedCmpXchg =
    Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
Value *Result = Builder.CreateCall(
    MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
if (XLen == 64)
  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
return Result;
6355}

6357bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                                   EVT VT) const {
VT = VT.getScalarType();

if (!VT.isSimple())
  return false;

switch (VT.getSimpleVT().SimpleTy) {
case MVT::f16:
  return Subtarget.hasStdExtZfh();
case MVT::f32:
  return Subtarget.hasStdExtF();
case MVT::f64:
  return Subtarget.hasStdExtD();
default:
  break;
}

return false;
6376}

6378Register RISCVTargetLowering::getExceptionPointerRegister(
  const Constant *PersonalityFn) const {
return RISCV::X10;
6381}

6383Register RISCVTargetLowering::getExceptionSelectorRegister(
  const Constant *PersonalityFn) const {
return RISCV::X11;
6386}

6388bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
// Return false to suppress the unnecessary extensions if the LibCall
// arguments or return value is f32 type for LP64 ABI.
RISCVABI::ABI ABI = Subtarget.getTargetABI();
if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
  return false;

return true;
6396}

6398bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
if (Subtarget.is64Bit() && Type == MVT::i32)
  return true;

return IsSigned;
6403}

6405bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
                                               SDValue C) const {
// Check integral scalar types.
if (VT.isScalarInteger()) {
  // Omit the optimization if the sub target has the M extension and the data
  // size exceeds XLen.
  if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
    return false;
  if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
    // Break the MUL to a SLLI and an ADD/SUB.
    const APInt &Imm = ConstNode->getAPIntValue();
    if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
        (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
      return true;
    // Omit the following optimization if the sub target has the M extension
    // and the data size >= XLen.
    if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
      return false;
    // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
    // a pair of LUI/ADDI.
    if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
      APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
      if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
          (1 - ImmS).isPowerOf2())
      return true;
    }
  }
}

return false;
6435}

6437bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
if (!Subtarget.useRVVForFixedLengthVectors())
  return false;

if (!VT.isFixedLengthVector())
  return false;

// Don't use RVV for vectors we cannot scalarize if required.
switch (VT.getVectorElementType().SimpleTy) {
// i1 is supported but has different rules.
default:
  return false;
case MVT::i1:
  // Masks can only use a single register.
  if (VT.getVectorNumElements() > Subtarget.getMinRVVVectorSizeInBits())
    return false;
  break;
case MVT::i8:
case MVT::i16:
case MVT::i32:
case MVT::i64:
  break;
case MVT::f16:
  if (!Subtarget.hasStdExtZfh())
    return false;
  break;
case MVT::f32:
  if (!Subtarget.hasStdExtF())
    return false;
  break;
case MVT::f64:
  if (!Subtarget.hasStdExtD())
    return false;
  break;
}

unsigned LMul = Subtarget.getLMULForFixedLengthVector(VT);
// Don't use RVV for types that don't fit.
if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
  return false;

// TODO: Perhaps an artificial restriction, but worth having whilst getting
// the base fixed length RVV support in place.
if (!VT.isPow2VectorType())
  return false;

return true;
6484}

6486bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
  EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
  bool *Fast) const {
if (!VT.isScalableVector())
  return false;

EVT ElemVT = VT.getVectorElementType();
if (Alignment >= ElemVT.getStoreSize()) {
  if (Fast)
    *Fast = true;
  return true;
}

return false;
6500}

6502#define GET_REGISTER_MATCHER
6503#include "RISCVGenAsmMatcher.inc"

6505Register
6506RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
                                     const MachineFunction &MF) const {
Register Reg = MatchRegisterAltName(RegName);
if (Reg == RISCV::NoRegister)
  Reg = MatchRegisterName(RegName);
if (Reg == RISCV::NoRegister)
  report_fatal_error(
      Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
  report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
                           StringRef(RegName) + "\"."));
return Reg;
6519}

6521namespace llvm {
6522namespace RISCVVIntrinsicsTable {

6524#define GET_RISCVVIntrinsicsTable_IMPL
6525#include "RISCVGenSearchableTables.inc"

6527} // namespace RISCVVIntrinsicsTable

6529} // namespace llvm

←

/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h

→

1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
10// and dyn_cast_or_null<X>() templates.
11//
12//===----------------------------------------------------------------------===//
13 
14#ifndef LLVM_SUPPORT_CASTING_H
15#define LLVM_SUPPORT_CASTING_H
16 
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/type_traits.h"
19#include <cassert>
20#include <memory>
21#include <type_traits>
22 
23namespace llvm {
24 
25//===----------------------------------------------------------------------===//
26//                          isa<x> Support Templates
27//===----------------------------------------------------------------------===//
28 
29// Define a template that can be specialized by smart pointers to reflect the
30// fact that they are automatically dereferenced, and are not involved with the
31// template selection process...  the default implementation is a noop.
32//
33template<typename From> struct simplify_type {
34  using SimpleType = From; // The real type this represents...
35 
36  // An accessor to get the real value...
37  static SimpleType &getSimplifiedValue(From &Val) { return Val; }
38};
39 
40template<typename From> struct simplify_type<const From> {
41  using NonConstSimpleType = typename simplify_type<From>::SimpleType;
42  using SimpleType =
43      typename add_const_past_pointer<NonConstSimpleType>::type;
44  using RetType =
45      typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
46 
47  static RetType getSimplifiedValue(const From& Val) {
48    return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
49  }
50};
51 
52// The core of the implementation of isa<X> is here; To and From should be
53// the names of classes.  This template can be specialized to customize the
54// implementation of isa<> without rewriting it from scratch.
55template <typename To, typename From, typename Enabler = void>
56struct isa_impl {
57  static inline bool doit(const From &Val) {
58    return To::classof(&Val);
59  }
60};
61 
62/// Always allow upcasts, and perform no dynamic check for them.
63template <typename To, typename From>
64struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
65  static inline bool doit(const From &) { return true; }
66};
67 
68template <typename To, typename From> struct isa_impl_cl {
69  static inline bool doit(const From &Val) {
70    return isa_impl<To, From>::doit(Val);
71  }
72};
73 
74template <typename To, typename From> struct isa_impl_cl<To, const From> {
75  static inline bool doit(const From &Val) {
76    return isa_impl<To, From>::doit(Val);
77  }
78};
79 
80template <typename To, typename From>
81struct isa_impl_cl<To, const std::unique_ptr<From>> {
82  static inline bool doit(const std::unique_ptr<From> &Val) {
83    assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 83, __PRETTY_FUNCTION__));
84    return isa_impl_cl<To, From>::doit(*Val);
85  }
86};
87 
88template <typename To, typename From> struct isa_impl_cl<To, From*> {
89  static inline bool doit(const From *Val) {
90    assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 90, __PRETTY_FUNCTION__));
91    return isa_impl<To, From>::doit(*Val);
92  }
93};
94 
95template <typename To, typename From> struct isa_impl_cl<To, From*const> {
96  static inline bool doit(const From *Val) {
97    assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 97, __PRETTY_FUNCTION__));
98    return isa_impl<To, From>::doit(*Val);
99  }
100};
101 
102template <typename To, typename From> struct isa_impl_cl<To, const From*> {
103  static inline bool doit(const From *Val) {
104    assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 104, __PRETTY_FUNCTION__));
105    return isa_impl<To, From>::doit(*Val);
106  }
107};
108 
109template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
110  static inline bool doit(const From *Val) {
111    assert(Val && "isa<> used on a null pointer")((Val && "isa<> used on a null pointer") ? static_cast
<void> (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 111, __PRETTY_FUNCTION__));
112    return isa_impl<To, From>::doit(*Val);
113  }
114};
115 
116template<typename To, typename From, typename SimpleFrom>
117struct isa_impl_wrap {
118  // When From != SimplifiedType, we can simplify the type some more by using
119  // the simplify_type template.
120  static bool doit(const From &Val) {
121    return isa_impl_wrap<To, SimpleFrom,
122      typename simplify_type<SimpleFrom>::SimpleType>::doit(
123                          simplify_type<const From>::getSimplifiedValue(Val));
124  }
125};
126 
127template<typename To, typename FromTy>
128struct isa_impl_wrap<To, FromTy, FromTy> {
129  // When From == SimpleType, we are as simple as we are going to get.
130  static bool doit(const FromTy &Val) {
131    return isa_impl_cl<To,FromTy>::doit(Val);
132  }
133};
134 
135// isa<X> - Return true if the parameter to the template is an instance of one
136// of the template type arguments.  Used like this:
137//
138//  if (isa<Type>(myVal)) { ... }
139//  if (isa<Type0, Type1, Type2>(myVal)) { ... }
140//
141template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
142  return isa_impl_wrap<X, const Y,
143                       typename simplify_type<const Y>::SimpleType>::doit(Val);
144}
145 
146template <typename First, typename Second, typename... Rest, typename Y>
147LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
148  return isa<First>(Val) || isa<Second, Rest...>(Val);
149}
150 
151// isa_and_nonnull<X> - Functionally identical to isa, except that a null value
152// is accepted.
153//
154template <typename... X, class Y>
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa_and_nonnull(const Y &Val) {
156  if (!Val)
157    return false;
158  return isa<X...>(Val);
159}
160 
161//===----------------------------------------------------------------------===//
162//                          cast<x> Support Templates
163//===----------------------------------------------------------------------===//
164 
165template<class To, class From> struct cast_retty;
166 
167// Calculate what type the 'cast' function should return, based on a requested
168// type of To and a source type of From.
169template<class To, class From> struct cast_retty_impl {
170  using ret_type = To &;       // Normal case, return Ty&
171};
172template<class To, class From> struct cast_retty_impl<To, const From> {
173  using ret_type = const To &; // Normal case, return Ty&
174};
175 
176template<class To, class From> struct cast_retty_impl<To, From*> {
177  using ret_type = To *;       // Pointer arg case, return Ty*
178};
179 
180template<class To, class From> struct cast_retty_impl<To, const From*> {
181  using ret_type = const To *; // Constant pointer arg case, return const Ty*
182};
183 
184template<class To, class From> struct cast_retty_impl<To, const From*const> {
185  using ret_type = const To *; // Constant pointer arg case, return const Ty*
186};
187 
188template <class To, class From>
189struct cast_retty_impl<To, std::unique_ptr<From>> {
190private:
191  using PointerType = typename cast_retty_impl<To, From *>::ret_type;
192  using ResultType = std::remove_pointer_t<PointerType>;
193 
194public:
195  using ret_type = std::unique_ptr<ResultType>;
196};
197 
198template<class To, class From, class SimpleFrom>
199struct cast_retty_wrap {
200  // When the simplified type and the from type are not the same, use the type
201  // simplifier to reduce the type, then reuse cast_retty_impl to get the
202  // resultant type.
203  using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
204};
205 
206template<class To, class FromTy>
207struct cast_retty_wrap<To, FromTy, FromTy> {
208  // When the simplified type is equal to the from type, use it directly.
209  using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
210};
211 
212template<class To, class From>
213struct cast_retty {
214  using ret_type = typename cast_retty_wrap<
215      To, From, typename simplify_type<From>::SimpleType>::ret_type;
216};
217 
218// Ensure the non-simple values are converted using the simplify_type template
219// that may be specialized by smart pointers...
220//
221template<class To, class From, class SimpleFrom> struct cast_convert_val {
222  // This is not a simple type, use the template to simplify it...
223  static typename cast_retty<To, From>::ret_type doit(From &Val) {
224    return cast_convert_val<To, SimpleFrom,
16
←
Returning without writing to 'Val.Node'→
225      typename simplify_type<SimpleFrom>::SimpleType>::doit(
226                          simplify_type<From>::getSimplifiedValue(Val));
13
←
Calling 'simplify_type::getSimplifiedValue'→
15
←
Returning from 'simplify_type::getSimplifiedValue'→
227  }
228};
229 
230template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
231  // This _is_ a simple type, just cast it.
232  static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
233    typename cast_retty<To, FromTy>::ret_type Res2
234     = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
235    return Res2;
236  }
237};
238 
239template <class X> struct is_simple_type {
240  static const bool value =
241      std::is_same<X, typename simplify_type<X>::SimpleType>::value;
242};
243 
244// cast<X> - Return the argument parameter cast to the specified type.  This
245// casting operator asserts that the type is correct, so it does not return null
246// on failure.  It does not allow a null argument (use cast_or_null for that).
247// It is typically used like this:
248//
249//  cast<Instruction>(myVal)->getParent()
250//
251template <class X, class Y>
252inline std::enable_if_t<!is_simple_type<Y>::value,
253                        typename cast_retty<X, const Y>::ret_type>
254cast(const Y &Val) {
255  assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 255, __PRETTY_FUNCTION__));
256  return cast_convert_val<
257      X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
258}
259 
260template <class X, class Y>
261inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
262  assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 262, __PRETTY_FUNCTION__));
10
←
Assuming 'Val' is a 'ConstantSDNode'→
11
←
'?' condition is true→
263  return cast_convert_val<X, Y,
12
←
Calling 'cast_convert_val::doit'→
17
←
Returning from 'cast_convert_val::doit'→
18
←
Returning without writing to 'Val.Node'→
264                          typename simplify_type<Y>::SimpleType>::doit(Val);
265}
266 
267template <class X, class Y>
268inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
269  assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 269, __PRETTY_FUNCTION__));
270  return cast_convert_val<X, Y*,
271                          typename simplify_type<Y*>::SimpleType>::doit(Val);
272}
273 
274template <class X, class Y>
275inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
276cast(std::unique_ptr<Y> &&Val) {
277  assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")((isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val.get()) && \"cast<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 277, __PRETTY_FUNCTION__));
278  using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
279  return ret_type(
280      cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
281          Val.release()));
282}
283 
284// cast_or_null<X> - Functionally identical to cast, except that a null value is
285// accepted.
286//
287template <class X, class Y>
288LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
289    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
290cast_or_null(const Y &Val) {
291  if (!Val)
292    return nullptr;
293  assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 293, __PRETTY_FUNCTION__));
294  return cast<X>(Val);
295}
296 
297template <class X, class Y>
298LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
299                                       typename cast_retty<X, Y>::ret_type>
300cast_or_null(Y &Val) {
301  if (!Val)
302    return nullptr;
303  assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 303, __PRETTY_FUNCTION__));
304  return cast<X>(Val);
305}
306 
307template <class X, class Y>
308LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
309cast_or_null(Y *Val) {
310  if (!Val) return nullptr;
311  assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")((isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? static_cast<void> (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/Support/Casting.h"
, 311, __PRETTY_FUNCTION__));
312  return cast<X>(Val);
313}
314 
315template <class X, class Y>
316inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
317cast_or_null(std::unique_ptr<Y> &&Val) {
318  if (!Val)
319    return nullptr;
320  return cast<X>(std::move(Val));
321}
322 
323// dyn_cast<X> - Return the argument parameter cast to the specified type.  This
324// casting operator returns null if the argument is of the wrong type, so it can
325// be used to test for a type as well as cast if successful.  This should be
326// used in the context of an if statement like this:
327//
328//  if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
329//
330 
331template <class X, class Y>
332LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
333    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
334dyn_cast(const Y &Val) {
335  return isa<X>(Val) ? cast<X>(Val) : nullptr;
336}
337 
338template <class X, class Y>
339LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
340  return isa<X>(Val) ? cast<X>(Val) : nullptr;
7
←
Assuming 'Val' is a 'ConstantSDNode'→
8
←
'?' condition is true→
9
←
Calling 'cast<llvm::ConstantSDNode, llvm::SDValue>'→
19
←
Returning from 'cast<llvm::ConstantSDNode, llvm::SDValue>'→
20
←
Returning without writing to 'Val.Node'→
341}
342 
343template <class X, class Y>
344LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
345  return isa<X>(Val) ? cast<X>(Val) : nullptr;
346}
347 
348// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
349// value is accepted.
350//
351template <class X, class Y>
352LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
353    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
354dyn_cast_or_null(const Y &Val) {
355  return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
356}
357 
358template <class X, class Y>
359LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
360                                       typename cast_retty<X, Y>::ret_type>
361dyn_cast_or_null(Y &Val) {
362  return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
363}
364 
365template <class X, class Y>
366LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
367dyn_cast_or_null(Y *Val) {
368  return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
369}
370 
371// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
372// taking ownership of the input pointer iff isa<X>(Val) is true.  If the
373// cast is successful, From refers to nullptr on exit and the casted value
374// is returned.  If the cast is unsuccessful, the function returns nullptr
375// and From is unchanged.
376template <class X, class Y>
377LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
378    -> decltype(cast<X>(Val)) {
379  if (!isa<X>(Val))
380    return nullptr;
381  return cast<X>(std::move(Val));
382}
383 
384template <class X, class Y>
385LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
386  return unique_dyn_cast<X, Y>(Val);
387}
388 
389// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
390// a null value is accepted.
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
393    -> decltype(cast<X>(Val)) {
394  if (!Val)
395    return nullptr;
396  return unique_dyn_cast<X, Y>(Val);
397}
398 
399template <class X, class Y>
400LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
401  return unique_dyn_cast_or_null<X, Y>(Val);
402}
403 
404} // end namespace llvm
405 
406#endif // LLVM_SUPPORT_CASTING_H

←

/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>

57namespace llvm {

59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;

72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG.  Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
82};

84namespace ISD {

/// Node predicates

88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);

109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);

113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);

117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);

125} // end namespace ISD

127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation.  Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node.  This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
unsigned ResNo = 0;     // Which return value of the node we are using.

144public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
230};

232template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
253};

255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
14
←
Returning without writing to 'Val.Node'→
}
263};
264template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
270};

272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

287public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

327private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
356};

358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
366};

368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

391public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
451};

453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
/// The operation that this node performs.
int16_t NodeType;

460protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with NodeType.

465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__ibmxl__))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif

475BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

571private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

608public:
/// Unique and persistent id per SDNode in the DAG.
/// Used for debug printing.
uint16_t PersistentId;

//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned short)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")((isMachineOpcode() && "Not a MachineInstr opcode!") ?
 static_cast<void> (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 679, __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator
  : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using reference = std::iterator<std::forward_iterator_tag,
                                  SDUse, ptrdiff_t>::reference;
  using pointer = std::iterator<std::forward_iterator_tag,
                                SDUse, ptrdiff_t>::pointer;

  use_iterator() = default;
  use_iterator(const use_iterator &I) : Op(I.Op) {}

  bool operator==(const use_iterator &x) const {
    return Op == x.Op;
  }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")((Op && "Cannot increment end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 748, __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 759, __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")((Num < NumOperands && "Invalid child # of SDNode!"
) ? static_cast<void> (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 897, __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")((ResNo < NumValues && "Illegal result number!") ?
 static_cast<void> (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 959, __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1046protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor"
) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1060, __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1067};

1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
DebugLoc DL;
int IROrder = 0;

1083public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")((Order >= 0 && "bad IROrder") ? static_cast<void
> (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1088, __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1095};

1097// Define inline functions from the SDValue class.

1099inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
 "Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
 "Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? static_cast<void> (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1106, __PRETTY_FUNCTION__));
1107}

1109inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
25
←
Called C++ object pointer is null
1111}

1113inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
1115}

1117inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1119}

1121inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1123}

1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1127}

1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1131}

1133inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1135}

1137inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1139}

1141inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1143}

1145inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1147}

1149inline bool SDValue::isUndef() const {
return Node->isUndef();
1151}

1153inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1155}

1157inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
1159}

1161inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1163}

1165inline void SDValue::dump() const {
return Node->dump();
1167}

1169inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1171}

1173inline void SDValue::dumpr() const {
return Node->dumpr();
1175}

1177inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1179}

1181// Define inline functions from the SDUse class.

1183inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode()) V.getNode()->addUse(*this);
1187}

1189inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V.getNode()->addUse(*this);
1192}

1194inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1198}

1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand.  This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
SDUse Op;

1207public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1226};

1228class AddrSpaceCastSDNode : public SDNode {
1229private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1233public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1243};

1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
// VT of in-memory value.
EVT MemoryVT;

1251protected:
/// Memory reference information.
MachineMemOperand *MMO;

1255public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }
LLVM_ATTRIBUTE_DEPRECATED(unsigned getOriginalAlignment() const,[[deprecated("Use getOriginalAlign() instead")]] unsigned getOriginalAlignment
() const
                          "Use getOriginalAlign() instead")[[deprecated("Use getOriginalAlign() instead")]] unsigned getOriginalAlignment
() const {
  return MMO->getBaseAlign().value();
}
// FIXME: Remove once transition to getAlign is over.
unsigned getAlignment() const { return MMO->getAlign().value(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getOrdering() const { return MMO->getOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::MSTORE:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  return N->getOpcode() == ISD::LOAD                ||
         N->getOpcode() == ISD::STORE               ||
         N->getOpcode() == ISD::PREFETCH            ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE        ||
         N->getOpcode() == ISD::MLOAD               ||
         N->getOpcode() == ISD::MSTORE              ||
         N->getOpcode() == ISD::MGATHER             ||
         N->getOpcode() == ISD::MSCATTER            ||
         N->isMemIntrinsic()                        ||
         N->isTargetMemoryOpcode();
}
1395};

1397/// This is an SDNode representing atomic operations.
1398class AtomicSDNode : public MemSDNode {
1399public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1404, __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1404, __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")((isCompareAndSwap() && "Must be cmpxchg operation") ?
 static_cast<void> (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1421, __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1446};

1448/// This SDNode is used for target intrinsics that touch
1449/// memory and need an associated MachineMemOperand. Its opcode may be
1450/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1451/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1452class MemIntrinsicSDNode : public MemSDNode {
1453public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1468};

1470/// This SDNode is used to implement the code generator
1471/// support for the llvm IR shufflevector instruction.  It combines elements
1472/// from two input vectors into a new input vector, with the selection and
1473/// ordering of elements determined by an array of integers, referred to as
1474/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1475/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1476/// An index of -1 is treated as undef, such that the code generator may put
1477/// any value in the corresponding element of the result.
1478class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1483protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1489public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return makeArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((Idx < getValueType(0).getVectorNumElements() && "Idx out of range!"
) ? static_cast<void> (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1496, __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")((isSplat() && "Cannot get splat index for non-splat!"
) ? static_cast<void> (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1503, __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1534};

1536class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1548public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isNullValue() const { return Value->isZero(); }
bool isAllOnesValue() const { return Value->isMinusOne(); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1569};

1571uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1573}

1575const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1577}

1579class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1589public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1624};

1626/// Returns true if \p V is a constant integer zero.
1627bool isNullConstant(SDValue V);

1629/// Returns true if \p V is an FP constant with a value of positive zero.
1630bool isNullFPConstant(SDValue V);

1632/// Returns true if \p V is an integer constant with all bits set.
1633bool isAllOnesConstant(SDValue V);

1635/// Returns true if \p V is a constant integer one.
1636bool isOneConstant(SDValue V);

1638/// Return the non-bitcasted source operand of \p V if it exists.
1639/// If \p V is not a bitcasted value, it is returned as-is.
1640SDValue peekThroughBitcasts(SDValue V);

1642/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1643/// If \p V is not a bitcasted one-use value, it is returned as-is.
1644SDValue peekThroughOneUseBitcasts(SDValue V);

1646/// Return the non-extracted vector source operand of \p V if it exists.
1647/// If \p V is not an extracted subvector, it is returned as-is.
1648SDValue peekThroughExtractSubvectors(SDValue V);

1650/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1651/// constant is canonicalized to be operand 1.
1652bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1654/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1655ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1658/// Returns the SDNode if it is a demanded constant splat BuildVector or
1659/// constant int.
1660ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1664/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1665ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1667/// Returns the SDNode if it is a demanded constant splat BuildVector or
1668/// constant float.
1669ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1672/// Return true if the value is a constant 0 integer or a splatted vector of
1673/// a constant 0 integer (with no undefs by default).
1674/// Build vector implicit truncation is not an issue for null values.
1675bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1677/// Return true if the value is a constant 1 integer or a splatted vector of a
1678/// constant 1 integer (with no undefs).
1679/// Does not permit build vector implicit truncation.
1680bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1682/// Return true if the value is a constant -1 integer or a splatted vector of a
1683/// constant -1 integer (with no undefs).
1684/// Does not permit build vector implicit truncation.
1685bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1687class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1698public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1711};

1713class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1723public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1730};

1732/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1733/// the offet and size that are started/ended in the underlying FrameIndex.
1734class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1742public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast<
void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1749, __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast<
void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1753, __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1762};

1764/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1765/// the index of the basic block being probed. A pseudo probe serves as a place
1766/// holder and will be removed at the end of compilation. It does not have any
1767/// operand because we do not want the instruction selection to deal with any.
1768class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1779public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1788};

1790class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1801public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1809};

1811class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1827, __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1836, __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1841public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((!isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1847, __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1852, __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1871};

1873/// Completely target-dependent object reference.
1874class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1881public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1893};

1895class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1907public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1913};

1915/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1916class BuildVectorSDNode : public SDNode {
1917public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

bool isConstant() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2026};

2028/// An SDNode that holds an arbitrary LLVM IR Value. This is
2029/// used when the SelectionDAG needs to make a simple reference to something
2030/// in the LLVM IR representation.
2031///
2032class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2041public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2048};

2050class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2059public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2065};

2067class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2075public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2081};

2083class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2093public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2099};

2101class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2113public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2122};

2124class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")((LabelSDNode::classof(this) && "not a label opcode")
 ? static_cast<void> (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2131, __PRETTY_FUNCTION__));
}

2134public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2141};

2143class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2154public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2162};

2164class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2172public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2178};

2180class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2189public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2195};

2197/// This class is used to represent EVT's, which are used
2198/// to parameterize some operations.
2199class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2208public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2214};

2216/// Base class for LoadSDNode and StoreSDNode
2217class LSBaseSDNode : public MemSDNode {
2218public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")((getAddressingMode() == AM && "Value truncated") ? static_cast
<void> (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2224, __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2247};

2249/// This class is used to represent ISD::LOAD nodes.
2250class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")((readMem() && "Load MachineMemOperand is not a load!"
) ? static_cast<void> (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2258, __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")((!writeMem() && "Load MachineMemOperand is a store!"
) ? static_cast<void> (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2259, __PRETTY_FUNCTION__));
}

2262public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2275};

2277/// This class is used to represent ISD::STORE nodes.
2278class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")((!readMem() && "Store MachineMemOperand is a load!")
 ? static_cast<void> (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2286, __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")((writeMem() && "Store MachineMemOperand is not a store!"
) ? static_cast<void> (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2287, __PRETTY_FUNCTION__));
}

2290public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2306};

2308/// This base class is used to represent MLOAD and MSTORE nodes
2309class MaskedLoadStoreSDNode : public MemSDNode {
2310public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")((getAddressingMode() == AM && "Value truncated") ? static_cast
<void> (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2319, __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2348};

2350/// This class is used to represent an MLOAD node
2351class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2352public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2377};

2379/// This class is used to represent an MSTORE node
2380class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2381public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2411};

2413/// This is a base class used to represent
2414/// MGATHER and MSCATTER nodes
2415///
2416class MaskedGatherScatterSDNode : public MemSDNode {
2417public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")((getIndexType() == IndexType && "Value truncated") ?
 static_cast<void> (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2425, __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
void setIndexType(ISD::MemIndexType IndexType) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2457};

2459/// This class is used to represent an MGATHER node
2460///
2461class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2462public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2482};

2484/// This class is used to represent an MSCATTER node
2485///
2486class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2487public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2508};

2510/// An SDNode that represents everything that will be needed
2511/// to construct a MachineInstr. These nodes are created during the
2512/// instruction selection proper phase.
2513///
2514/// Note that the only supported way to set the `memoperands` is by calling the
2515/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2516/// inside the DAG rather than in the node.
2517class MachineSDNode : public SDNode {
2518private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2546public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2572};

2574/// An SDNode that records if a register contains a value that is guaranteed to
2575/// be aligned accordingly.
2576class AssertAlignSDNode : public SDNode {
Align Alignment;

2579public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2588};

2590class SDNodeIterator : public std::iterator<std::forward_iterator_tag,
                                          SDNode, ptrdiff_t> {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2597public:
bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&((Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? static_cast<void> (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2617, __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")((Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? static_cast<void> (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-13~++20210310111112+6f912a2cd491/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2617, __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
2628};

2630template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
2643};

2645/// A representation of the largest SDNode, for use in sizeof().
2646///
2647/// This needs to be a union because the largest node differs on 32 bit systems
2648/// with 4 and 8 byte pointer alignment, respectively.
2649using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

2654/// The SDNode class with the greatest alignment requirement.
2655using MostAlignedSDNode = GlobalAddressSDNode;

2657namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating store.
inline bool isNON_TRUNCStore(const SDNode *N) {
  return isa<StoreSDNode>(N) && !cast<StoreSDNode>(N)->isTruncatingStore();
}

/// Returns true if the specified node is a truncating store.
inline bool isTRUNCStore(const SDNode *N) {
  return isa<StoreSDNode>(N) && cast<StoreSDNode>(N)->isTruncatingStore();
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

2745} // end namespace ISD

2747} // end namespace llvm

2749#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H