doxygen/LoongArchISelLowering_8cpp_source.html

//=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation  ---===//

//

// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

// See https://llvm.org/LICENSE.txt for license information.

// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

//

//===----------------------------------------------------------------------===//

//

// This file defines the interfaces that LoongArch uses to lower LLVM code into

// a selection DAG.

//

//===----------------------------------------------------------------------===//


#include "LoongArchISelLowering.h"

#include "LoongArch.h"

#include "LoongArchMachineFunctionInfo.h"

#include "LoongArchRegisterInfo.h"

#include "LoongArchSubtarget.h"

#include "MCTargetDesc/LoongArchBaseInfo.h"

#include "MCTargetDesc/LoongArchMCTargetDesc.h"

#include "llvm/ADT/Statistic.h"

#include "llvm/ADT/StringExtras.h"

#include "llvm/CodeGen/ISDOpcodes.h"

#include "llvm/CodeGen/RuntimeLibcallUtil.h"

#include "llvm/CodeGen/SelectionDAGNodes.h"

#include "llvm/IR/IRBuilder.h"

#include "llvm/IR/IntrinsicInst.h"

#include "llvm/IR/IntrinsicsLoongArch.h"

#include "llvm/Support/CodeGen.h"

#include "llvm/Support/Debug.h"

#include "llvm/Support/ErrorHandling.h"

#include "llvm/Support/KnownBits.h"

#include "llvm/Support/MathExtras.h"


using namespace llvm;


#define DEBUG_TYPE "loongarch-isel-lowering"


STATISTIC(NumTailCalls, "Number of tail calls");


static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,

                                  cl::desc("Trap on integer division by zero."),

                                  cl::init(false));


LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,

                                                 const LoongArchSubtarget &STI)

    : TargetLowering(TM), Subtarget(STI) {


  MVT GRLenVT = Subtarget.getGRLenVT();


  // Set up the register classes.


  addRegisterClass(GRLenVT, &LoongArch::GPRRegClass);

  if (Subtarget.hasBasicF())

    addRegisterClass(MVT::f32, &LoongArch::FPR32RegClass);

  if (Subtarget.hasBasicD())

    addRegisterClass(MVT::f64, &LoongArch::FPR64RegClass);


  static const MVT::SimpleValueType LSXVTs[] = {

      MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};

  static const MVT::SimpleValueType LASXVTs[] = {

      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};


  if (Subtarget.hasExtLSX())

    for (MVT VT : LSXVTs)

      addRegisterClass(VT, &LoongArch::LSX128RegClass);


  if (Subtarget.hasExtLASX())

    for (MVT VT : LASXVTs)

      addRegisterClass(VT, &LoongArch::LASX256RegClass);


  // Set operations for LA32 and LA64.


  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, GRLenVT,

                   MVT::i1, Promote);


  setOperationAction(ISD::SHL_PARTS, GRLenVT, Custom);

  setOperationAction(ISD::SRA_PARTS, GRLenVT, Custom);

  setOperationAction(ISD::SRL_PARTS, GRLenVT, Custom);

  setOperationAction(ISD::FP_TO_SINT, GRLenVT, Custom);

  setOperationAction(ISD::ROTL, GRLenVT, Expand);

  setOperationAction(ISD::CTPOP, GRLenVT, Expand);


  setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,

                      ISD::JumpTable, ISD::GlobalTLSAddress},

                     GRLenVT, Custom);


  setOperationAction(ISD::EH_DWARF_CFA, GRLenVT, Custom);


  setOperationAction(ISD::DYNAMIC_STACKALLOC, GRLenVT, Expand);

  setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);

  setOperationAction(ISD::VASTART, MVT::Other, Custom);

  setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);


  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);

  setOperationAction(ISD::TRAP, MVT::Other, Legal);


  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);

  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);

  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);


  // Expand bitreverse.i16 with native-width bitrev and shift for now, before

  // we get to know which of sll and revb.2h is faster.

  setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);

  setOperationAction(ISD::BITREVERSE, GRLenVT, Legal);


  // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and

  // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16

  // and i32 could still be byte-swapped relatively cheaply.

  setOperationAction(ISD::BSWAP, MVT::i16, Custom);


  setOperationAction(ISD::BR_JT, MVT::Other, Expand);

  setOperationAction(ISD::BR_CC, GRLenVT, Expand);

  setOperationAction(ISD::SELECT_CC, GRLenVT, Expand);

  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);

  setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, GRLenVT, Expand);


  setOperationAction(ISD::FP_TO_UINT, GRLenVT, Custom);

  setOperationAction(ISD::UINT_TO_FP, GRLenVT, Expand);


  // Set operations for LA64 only.


  if (Subtarget.is64Bit()) {

    setOperationAction(ISD::ADD, MVT::i32, Custom);

    setOperationAction(ISD::SUB, MVT::i32, Custom);

    setOperationAction(ISD::SHL, MVT::i32, Custom);

    setOperationAction(ISD::SRA, MVT::i32, Custom);

    setOperationAction(ISD::SRL, MVT::i32, Custom);

    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);

    setOperationAction(ISD::BITCAST, MVT::i32, Custom);

    setOperationAction(ISD::ROTR, MVT::i32, Custom);

    setOperationAction(ISD::ROTL, MVT::i32, Custom);

    setOperationAction(ISD::CTTZ, MVT::i32, Custom);

    setOperationAction(ISD::CTLZ, MVT::i32, Custom);

    setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);

    setOperationAction(ISD::READ_REGISTER, MVT::i32, Custom);

    setOperationAction(ISD::WRITE_REGISTER, MVT::i32, Custom);

    setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom);

    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);

    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);


    setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);

    setOperationAction(ISD::BSWAP, MVT::i32, Custom);

    setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, MVT::i32,

                       Custom);

    setOperationAction(ISD::LROUND, MVT::i32, Custom);

  }


  // Set operations for LA32 only.


  if (!Subtarget.is64Bit()) {

    setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);

    setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);

    setOperationAction(ISD::INTRINSIC_VOID, MVT::i64, Custom);

    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);

    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);

  }


  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);


  static const ISD::CondCode FPCCToExpand[] = {

      ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,

      ISD::SETGE,  ISD::SETNE,  ISD::SETGT};


  // Set operations for 'F' feature.


  if (Subtarget.hasBasicF()) {

    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);

    setTruncStoreAction(MVT::f32, MVT::f16, Expand);

    setCondCodeAction(FPCCToExpand, MVT::f32, Expand);


    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);

    setOperationAction(ISD::BR_CC, MVT::f32, Expand);

    setOperationAction(ISD::FMA, MVT::f32, Legal);

    setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal);

    setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);

    setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);

    setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);

    setOperationAction(ISD::IS_FPCLASS, MVT::f32, Legal);

    setOperationAction(ISD::FSIN, MVT::f32, Expand);

    setOperationAction(ISD::FCOS, MVT::f32, Expand);

    setOperationAction(ISD::FSINCOS, MVT::f32, Expand);

    setOperationAction(ISD::FPOW, MVT::f32, Expand);

    setOperationAction(ISD::FREM, MVT::f32, Expand);

    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);

    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);


    if (Subtarget.is64Bit())

      setOperationAction(ISD::FRINT, MVT::f32, Legal);


    if (!Subtarget.hasBasicD()) {

      setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);

      if (Subtarget.is64Bit()) {

        setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);

        setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);

      }

    }

  }


  // Set operations for 'D' feature.


  if (Subtarget.hasBasicD()) {

    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);

    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);

    setTruncStoreAction(MVT::f64, MVT::f16, Expand);

    setTruncStoreAction(MVT::f64, MVT::f32, Expand);

    setCondCodeAction(FPCCToExpand, MVT::f64, Expand);


    setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);

    setOperationAction(ISD::BR_CC, MVT::f64, Expand);

    setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);

    setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);

    setOperationAction(ISD::FMA, MVT::f64, Legal);

    setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);

    setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal);

    setOperationAction(ISD::IS_FPCLASS, MVT::f64, Legal);

    setOperationAction(ISD::FSIN, MVT::f64, Expand);

    setOperationAction(ISD::FCOS, MVT::f64, Expand);

    setOperationAction(ISD::FSINCOS, MVT::f64, Expand);

    setOperationAction(ISD::FPOW, MVT::f64, Expand);

    setOperationAction(ISD::FREM, MVT::f64, Expand);

    setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);

    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);


    if (Subtarget.is64Bit())

      setOperationAction(ISD::FRINT, MVT::f64, Legal);

  }


  // Set operations for 'LSX' feature.


  if (Subtarget.hasExtLSX()) {

    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {

      // Expand all truncating stores and extending loads.

      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {

        setTruncStoreAction(VT, InnerVT, Expand);

        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);

        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);

        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);

      }

      // By default everything must be expanded. Then we will selectively turn

      // on ones that can be effectively codegen'd.

      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)

        setOperationAction(Op, VT, Expand);

    }


    for (MVT VT : LSXVTs) {

      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Legal);

      setOperationAction(ISD::BITCAST, VT, Legal);

      setOperationAction(ISD::UNDEF, VT, Legal);


      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);

      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal);

      setOperationAction(ISD::BUILD_VECTOR, VT, Custom);


      setOperationAction(ISD::SETCC, VT, Legal);

      setOperationAction(ISD::VSELECT, VT, Legal);

      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);

    }

    for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {

      setOperationAction({ISD::ADD, ISD::SUB}, VT, Legal);

      setOperationAction({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,

                         Legal);

      setOperationAction({ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},

                         VT, Legal);

      setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Legal);

      setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL}, VT, Legal);

      setOperationAction({ISD::CTPOP, ISD::CTLZ}, VT, Legal);

      setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal);

      setCondCodeAction(

          {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,

          Expand);

    }

    for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})

      setOperationAction(ISD::BITREVERSE, VT, Custom);

    for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})

      setOperationAction(ISD::BSWAP, VT, Legal);

    for (MVT VT : {MVT::v4i32, MVT::v2i64}) {

      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Legal);

      setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Legal);

    }

    for (MVT VT : {MVT::v4f32, MVT::v2f64}) {

      setOperationAction({ISD::FADD, ISD::FSUB}, VT, Legal);

      setOperationAction({ISD::FMUL, ISD::FDIV}, VT, Legal);

      setOperationAction(ISD::FMA, VT, Legal);

      setOperationAction(ISD::FSQRT, VT, Legal);

      setOperationAction(ISD::FNEG, VT, Legal);

      setCondCodeAction({ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,

                         ISD::SETUGE, ISD::SETUGT},

                        VT, Expand);

    }

    setOperationAction(ISD::CTPOP, GRLenVT, Legal);

    setOperationAction(ISD::FCEIL, {MVT::f32, MVT::f64}, Legal);

    setOperationAction(ISD::FFLOOR, {MVT::f32, MVT::f64}, Legal);

    setOperationAction(ISD::FTRUNC, {MVT::f32, MVT::f64}, Legal);

    setOperationAction(ISD::FROUNDEVEN, {MVT::f32, MVT::f64}, Legal);

  }


  // Set operations for 'LASX' feature.


  if (Subtarget.hasExtLASX()) {

    for (MVT VT : LASXVTs) {

      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Legal);

      setOperationAction(ISD::BITCAST, VT, Legal);

      setOperationAction(ISD::UNDEF, VT, Legal);


      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);

      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);

      setOperationAction(ISD::BUILD_VECTOR, VT, Custom);

      setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);


      setOperationAction(ISD::SETCC, VT, Legal);

      setOperationAction(ISD::VSELECT, VT, Legal);

      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);

    }

    for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {

      setOperationAction({ISD::ADD, ISD::SUB}, VT, Legal);

      setOperationAction({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN}, VT,

                         Legal);

      setOperationAction({ISD::MUL, ISD::SDIV, ISD::SREM, ISD::UDIV, ISD::UREM},

                         VT, Legal);

      setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Legal);

      setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL}, VT, Legal);

      setOperationAction({ISD::CTPOP, ISD::CTLZ}, VT, Legal);

      setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal);

      setCondCodeAction(

          {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT,

          Expand);

    }

    for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})

      setOperationAction(ISD::BITREVERSE, VT, Custom);

    for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})

      setOperationAction(ISD::BSWAP, VT, Legal);

    for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {

      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Legal);

      setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT, Legal);

    }

    for (MVT VT : {MVT::v8f32, MVT::v4f64}) {

      setOperationAction({ISD::FADD, ISD::FSUB}, VT, Legal);

      setOperationAction({ISD::FMUL, ISD::FDIV}, VT, Legal);

      setOperationAction(ISD::FMA, VT, Legal);

      setOperationAction(ISD::FSQRT, VT, Legal);

      setOperationAction(ISD::FNEG, VT, Legal);

      setCondCodeAction({ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT,

                         ISD::SETUGE, ISD::SETUGT},

                        VT, Expand);

    }

  }


  // Set DAG combine for LA32 and LA64.


  setTargetDAGCombine(ISD::AND);

  setTargetDAGCombine(ISD::OR);

  setTargetDAGCombine(ISD::SRL);

  setTargetDAGCombine(ISD::SETCC);


  // Set DAG combine for 'LSX' feature.


  if (Subtarget.hasExtLSX())

    setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);


  // Compute derived properties from the register classes.

  computeRegisterProperties(Subtarget.getRegisterInfo());


  setStackPointerRegisterToSaveRestore(LoongArch::R3);


  setBooleanContents(ZeroOrOneBooleanContent);

  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);


  setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen());


  setMinCmpXchgSizeInBits(32);


  // Function alignments.

  setMinFunctionAlignment(Align(4));

  // Set preferred alignments.

  setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());

  setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());

  setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());


  // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.

  if (Subtarget.hasLAMCAS())

    setMinCmpXchgSizeInBits(8);

}


bool LoongArchTargetLowering::isOffsetFoldingLegal(

    const GlobalAddressSDNode *GA) const {

  // In order to maximise the opportunity for common subexpression elimination,

  // keep a separate ADD node for the global address offset instead of folding

  // it in the global address node. Later peephole optimisations may choose to

  // fold it back in when profitable.

  return false;

}


SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,

                                                SelectionDAG &DAG) const {

  switch (Op.getOpcode()) {

  case ISD::ATOMIC_FENCE:

    return lowerATOMIC_FENCE(Op, DAG);

  case ISD::EH_DWARF_CFA:

    return lowerEH_DWARF_CFA(Op, DAG);

  case ISD::GlobalAddress:

    return lowerGlobalAddress(Op, DAG);

  case ISD::GlobalTLSAddress:

    return lowerGlobalTLSAddress(Op, DAG);

  case ISD::INTRINSIC_WO_CHAIN:

    return lowerINTRINSIC_WO_CHAIN(Op, DAG);

  case ISD::INTRINSIC_W_CHAIN:

    return lowerINTRINSIC_W_CHAIN(Op, DAG);

  case ISD::INTRINSIC_VOID:

    return lowerINTRINSIC_VOID(Op, DAG);

  case ISD::BlockAddress:

    return lowerBlockAddress(Op, DAG);

  case ISD::JumpTable:

    return lowerJumpTable(Op, DAG);

  case ISD::SHL_PARTS:

    return lowerShiftLeftParts(Op, DAG);

  case ISD::SRA_PARTS:

    return lowerShiftRightParts(Op, DAG, true);

  case ISD::SRL_PARTS:

    return lowerShiftRightParts(Op, DAG, false);

  case ISD::ConstantPool:

    return lowerConstantPool(Op, DAG);

  case ISD::FP_TO_SINT:

    return lowerFP_TO_SINT(Op, DAG);

  case ISD::BITCAST:

    return lowerBITCAST(Op, DAG);

  case ISD::UINT_TO_FP:

    return lowerUINT_TO_FP(Op, DAG);

  case ISD::SINT_TO_FP:

    return lowerSINT_TO_FP(Op, DAG);

  case ISD::VASTART:

    return lowerVASTART(Op, DAG);

  case ISD::FRAMEADDR:

    return lowerFRAMEADDR(Op, DAG);

  case ISD::RETURNADDR:

    return lowerRETURNADDR(Op, DAG);

  case ISD::WRITE_REGISTER:

    return lowerWRITE_REGISTER(Op, DAG);

  case ISD::INSERT_VECTOR_ELT:

    return lowerINSERT_VECTOR_ELT(Op, DAG);

  case ISD::EXTRACT_VECTOR_ELT:

    return lowerEXTRACT_VECTOR_ELT(Op, DAG);

  case ISD::BUILD_VECTOR:

    return lowerBUILD_VECTOR(Op, DAG);

  case ISD::VECTOR_SHUFFLE:

    return lowerVECTOR_SHUFFLE(Op, DAG);

  case ISD::BITREVERSE:

    return lowerBITREVERSE(Op, DAG);

  }

  return SDValue();

}


SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,

                                                 SelectionDAG &DAG) const {

  EVT ResTy = Op->getValueType(0);

  SDValue Src = Op->getOperand(0);

  SDLoc DL(Op);


  EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;

  unsigned int OrigEltNum = ResTy.getVectorNumElements();

  unsigned int NewEltNum = NewVT.getVectorNumElements();


  SDValue NewSrc = DAG.getNode(ISD::BITCAST, DL, NewVT, Src);


  SmallVector<SDValue, 8> Ops;

  for (unsigned int i = 0; i < NewEltNum; i++) {

    SDValue Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, NewSrc,

                             DAG.getConstant(i, DL, MVT::i64));

    SDValue RevOp = DAG.getNode((ResTy == MVT::v16i8 || ResTy == MVT::v32i8)

                                    ? LoongArchISD::BITREV_8B

                                    : ISD::BITREVERSE,

                                DL, MVT::i64, Op);

    Ops.push_back(RevOp);

  }

  SDValue Res =

      DAG.getNode(ISD::BITCAST, DL, ResTy, DAG.getBuildVector(NewVT, DL, Ops));


  switch (ResTy.getSimpleVT().SimpleTy) {

  default:

    return SDValue();

  case MVT::v16i8:

  case MVT::v32i8:

    return Res;

  case MVT::v8i16:

  case MVT::v16i16:

  case MVT::v4i32:

  case MVT::v8i32: {

    SmallVector<int, 32> Mask;

    for (unsigned int i = 0; i < NewEltNum; i++)

      for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)

        Mask.push_back(j + (OrigEltNum / NewEltNum) * i);

    return DAG.getVectorShuffle(ResTy, DL, Res, DAG.getUNDEF(ResTy), Mask);

  }

  }

}


/// Determine whether a range fits a regular pattern of values.

/// This function accounts for the possibility of jumping over the End iterator.

template <typename ValType>

static bool

fitsRegularPattern(typename SmallVectorImpl<ValType>::const_iterator Begin,

                   unsigned CheckStride,

                   typename SmallVectorImpl<ValType>::const_iterator End,

                   ValType ExpectedIndex, unsigned ExpectedIndexStride) {

  auto &I = Begin;


  while (I != End) {

    if (*I != -1 && *I != ExpectedIndex)

      return false;

    ExpectedIndex += ExpectedIndexStride;


    // Incrementing past End is undefined behaviour so we must increment one

    // step at a time and check for End at each step.

    for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)

      ; // Empty loop body.

  }

  return true;

}


/// Lower VECTOR_SHUFFLE into VREPLVEI (if possible).

///

/// VREPLVEI performs vector broadcast based on an element specified by an

/// integer immediate, with its mask being similar to:

///   <x, x, x, ...>

/// where x is any valid index.

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above form.

static SDValue lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef<int> Mask,

                                            MVT VT, SDValue V1, SDValue V2,

                                            SelectionDAG &DAG) {

  int SplatIndex = -1;

  for (const auto &M : Mask) {

    if (M != -1) {

      SplatIndex = M;

      break;

    }

  }


  if (SplatIndex == -1)

    return DAG.getUNDEF(VT);


  assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");

  if (fitsRegularPattern<int>(Mask.begin(), 1, Mask.end(), SplatIndex, 0)) {

    APInt Imm(64, SplatIndex);

    return DAG.getNode(LoongArchISD::VREPLVEI, DL, VT, V1,

                       DAG.getConstant(Imm, DL, MVT::i64));

  }


  return SDValue();

}


/// Lower VECTOR_SHUFFLE into VSHUF4I (if possible).

///

/// VSHUF4I splits the vector into blocks of four elements, then shuffles these

/// elements according to a <4 x i2> constant (encoded as an integer immediate).

///

/// It is therefore possible to lower into VSHUF4I when the mask takes the form:

///   <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>

/// When undef's appear they are treated as if they were whatever value is

/// necessary in order to fit the above forms.

///

/// For example:

///   %2 = shufflevector <8 x i16> %0, <8 x i16> undef,

///                      <8 x i32> <i32 3, i32 2, i32 1, i32 0,

///                                 i32 7, i32 6, i32 5, i32 4>

/// is lowered to:

///   (VSHUF4I_H $v0, $v1, 27)

/// where the 27 comes from:

///   3 + (2 << 2) + (1 << 4) + (0 << 6)

static SDValue lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef<int> Mask,

                                           MVT VT, SDValue V1, SDValue V2,

                                           SelectionDAG &DAG) {


  // When the size is less than 4, lower cost instructions may be used.

  if (Mask.size() < 4)

    return SDValue();


  int SubMask[4] = {-1, -1, -1, -1};

  for (unsigned i = 0; i < 4; ++i) {

    for (unsigned j = i; j < Mask.size(); j += 4) {

      int Idx = Mask[j];


      // Convert from vector index to 4-element subvector index

      // If an index refers to an element outside of the subvector then give up

      if (Idx != -1) {

        Idx -= 4 * (j / 4);

        if (Idx < 0 || Idx >= 4)

          return SDValue();

      }


      // If the mask has an undef, replace it with the current index.

      // Note that it might still be undef if the current index is also undef

      if (SubMask[i] == -1)

        SubMask[i] = Idx;

      // Check that non-undef values are the same as in the mask. If they

      // aren't then give up

      else if (Idx != -1 && Idx != SubMask[i])

        return SDValue();

    }

  }


  // Calculate the immediate. Replace any remaining undefs with zero

  APInt Imm(64, 0);

  for (int i = 3; i >= 0; --i) {

    int Idx = SubMask[i];


    if (Idx == -1)

      Idx = 0;


    Imm <<= 2;

    Imm |= Idx & 0x3;

  }


  return DAG.getNode(LoongArchISD::VSHUF4I, DL, VT, V1,

                     DAG.getConstant(Imm, DL, MVT::i64));

}


/// Lower VECTOR_SHUFFLE into VPACKEV (if possible).

///

/// VPACKEV interleaves the even elements from each vector.

///

/// It is possible to lower into VPACKEV when the mask consists of two of the

/// following forms interleaved:

///   <0, 2, 4, ...>

///   <n, n+2, n+4, ...>

/// where n is the number of elements in the vector.

/// For example:

///   <0, 0, 2, 2, 4, 4, ...>

///   <0, n, 2, n+2, 4, n+4, ...>

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above forms.

static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef<int> Mask,

                                           MVT VT, SDValue V1, SDValue V2,

                                           SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 2, End, 0, 2))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size(), 2))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Begin + 1, 2, End, 0, 2))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size(), 2))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VPACKEV, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into VPACKOD (if possible).

///

/// VPACKOD interleaves the odd elements from each vector.

///

/// It is possible to lower into VPACKOD when the mask consists of two of the

/// following forms interleaved:

///   <1, 3, 5, ...>

///   <n+1, n+3, n+5, ...>

/// where n is the number of elements in the vector.

/// For example:

///   <1, 1, 3, 3, 5, 5, ...>

///   <1, n+1, 3, n+3, 5, n+5, ...>

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above forms.

static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef<int> Mask,

                                           MVT VT, SDValue V1, SDValue V2,

                                           SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 2, End, 1, 2))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size() + 1, 2))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Begin + 1, 2, End, 1, 2))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size() + 1, 2))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VPACKOD, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into VILVH (if possible).

///

/// VILVH interleaves consecutive elements from the left (highest-indexed) half

/// of each vector.

///

/// It is possible to lower into VILVH when the mask consists of two of the

/// following forms interleaved:

///   <x, x+1, x+2, ...>

///   <n+x, n+x+1, n+x+2, ...>

/// where n is the number of elements in the vector and x is half n.

/// For example:

///   <x, x, x+1, x+1, x+2, x+2, ...>

///   <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above forms.

static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef<int> Mask,

                                         MVT VT, SDValue V1, SDValue V2,

                                         SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  unsigned HalfSize = Mask.size() / 2;

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 2, End, HalfSize, 1))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size() + HalfSize, 1))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Begin + 1, 2, End, HalfSize, 1))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size() + HalfSize,

                                   1))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VILVH, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into VILVL (if possible).

///

/// VILVL interleaves consecutive elements from the right (lowest-indexed) half

/// of each vector.

///

/// It is possible to lower into VILVL when the mask consists of two of the

/// following forms interleaved:

///   <0, 1, 2, ...>

///   <n, n+1, n+2, ...>

/// where n is the number of elements in the vector.

/// For example:

///   <0, 0, 1, 1, 2, 2, ...>

///   <0, n, 1, n+1, 2, n+2, ...>

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above forms.

static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef<int> Mask,

                                         MVT VT, SDValue V1, SDValue V2,

                                         SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 2, End, 0, 1))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size(), 1))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Begin + 1, 2, End, 0, 1))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size(), 1))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VILVL, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into VPICKEV (if possible).

///

/// VPICKEV copies the even elements of each vector into the result vector.

///

/// It is possible to lower into VPICKEV when the mask consists of two of the

/// following forms concatenated:

///   <0, 2, 4, ...>

///   <n, n+2, n+4, ...>

/// where n is the number of elements in the vector.

/// For example:

///   <0, 2, 4, ..., 0, 2, 4, ...>

///   <0, 2, 4, ..., n, n+2, n+4, ...>

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above forms.

static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef<int> Mask,

                                           MVT VT, SDValue V1, SDValue V2,

                                           SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &Mid = Mask.begin() + Mask.size() / 2;

  const auto &End = Mask.end();

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 1, Mid, 0, 2))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 1, Mid, Mask.size(), 2))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Mid, 1, End, 0, 2))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Mid, 1, End, Mask.size(), 2))

    V2 = OriV2;


  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VPICKEV, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into VPICKOD (if possible).

///

/// VPICKOD copies the odd elements of each vector into the result vector.

///

/// It is possible to lower into VPICKOD when the mask consists of two of the

/// following forms concatenated:

///   <1, 3, 5, ...>

///   <n+1, n+3, n+5, ...>

/// where n is the number of elements in the vector.

/// For example:

///   <1, 3, 5, ..., 1, 3, 5, ...>

///   <1, 3, 5, ..., n+1, n+3, n+5, ...>

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above forms.

static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef<int> Mask,

                                           MVT VT, SDValue V1, SDValue V2,

                                           SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &Mid = Mask.begin() + Mask.size() / 2;

  const auto &End = Mask.end();

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 1, Mid, 1, 2))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 1, Mid, Mask.size() + 1, 2))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Mid, 1, End, 1, 2))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Mid, 1, End, Mask.size() + 1, 2))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VPICKOD, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into VSHUF.

///

/// This mostly consists of converting the shuffle mask into a BUILD_VECTOR and

/// adding it as an operand to the resulting VSHUF.

static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef<int> Mask,

                                         MVT VT, SDValue V1, SDValue V2,

                                         SelectionDAG &DAG) {


  SmallVector<SDValue, 16> Ops;

  for (auto M : Mask)

    Ops.push_back(DAG.getConstant(M, DL, MVT::i64));


  EVT MaskVecTy = VT.changeVectorElementTypeToInteger();

  SDValue MaskVec = DAG.getBuildVector(MaskVecTy, DL, Ops);


  // VECTOR_SHUFFLE concatenates the vectors in an vectorwise fashion.

  // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11>

  // VSHF concatenates the vectors in a bitwise fashion:

  // <0b00, 0b01> + <0b10, 0b11> ->

  // 0b0100       + 0b1110       -> 0b01001110

  //                                <0b10, 0b11, 0b00, 0b01>

  // We must therefore swap the operands to get the correct result.

  return DAG.getNode(LoongArchISD::VSHUF, DL, VT, MaskVec, V2, V1);

}


/// Dispatching routine to lower various 128-bit LoongArch vector shuffles.

///

/// This routine breaks down the specific type of 128-bit shuffle and

/// dispatches to the lowering routines accordingly.

static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,

                                  SDValue V1, SDValue V2, SelectionDAG &DAG) {

  assert((VT.SimpleTy == MVT::v16i8 || VT.SimpleTy == MVT::v8i16 ||

          VT.SimpleTy == MVT::v4i32 || VT.SimpleTy == MVT::v2i64 ||

          VT.SimpleTy == MVT::v4f32 || VT.SimpleTy == MVT::v2f64) &&

         "Vector type is unsupported for lsx!");

  assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&

         "Two operands have different types!");

  assert(VT.getVectorNumElements() == Mask.size() &&

         "Unexpected mask size for shuffle!");

  assert(Mask.size() % 2 == 0 && "Expected even mask size.");


  SDValue Result;

  // TODO: Add more comparison patterns.

  if (V2.isUndef()) {

    if ((Result = lowerVECTOR_SHUFFLE_VREPLVEI(DL, Mask, VT, V1, V2, DAG)))

      return Result;

    if ((Result = lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG)))

      return Result;


    // TODO: This comment may be enabled in the future to better match the

    // pattern for instruction selection.

    /* V2 = V1; */

  }


  // It is recommended not to change the pattern comparison order for better

  // performance.

  if ((Result = lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_VILVH(DL, Mask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_VILVL(DL, Mask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_VPICKEV(DL, Mask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG)))

    return Result;


  return SDValue();

}


/// Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).

///

/// It is a XVREPLVEI when the mask is:

///   <x, x, x, ..., x+n, x+n, x+n, ...>

/// where the number of x is equal to n and n is half the length of vector.

///

/// When undef's appear in the mask they are treated as if they were whatever

/// value is necessary in order to fit the above form.

static SDValue lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL,

                                             ArrayRef<int> Mask, MVT VT,

                                             SDValue V1, SDValue V2,

                                             SelectionDAG &DAG) {

  int SplatIndex = -1;

  for (const auto &M : Mask) {

    if (M != -1) {

      SplatIndex = M;

      break;

    }

  }


  if (SplatIndex == -1)

    return DAG.getUNDEF(VT);


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  unsigned HalfSize = Mask.size() / 2;


  assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");

  if (fitsRegularPattern<int>(Begin, 1, End - HalfSize, SplatIndex, 0) &&

      fitsRegularPattern<int>(Begin + HalfSize, 1, End, SplatIndex + HalfSize,

                              0)) {

    APInt Imm(64, SplatIndex);

    return DAG.getNode(LoongArchISD::VREPLVEI, DL, VT, V1,

                       DAG.getConstant(Imm, DL, MVT::i64));

  }


  return SDValue();

}


/// Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef<int> Mask,

                                            MVT VT, SDValue V1, SDValue V2,

                                            SelectionDAG &DAG) {

  // When the size is less than or equal to 4, lower cost instructions may be

  // used.

  if (Mask.size() <= 4)

    return SDValue();

  return lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG);

}


/// Lower VECTOR_SHUFFLE into XVPACKEV (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef<int> Mask,

                                            MVT VT, SDValue V1, SDValue V2,

                                            SelectionDAG &DAG) {

  return lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG);

}


/// Lower VECTOR_SHUFFLE into XVPACKOD (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef<int> Mask,

                                            MVT VT, SDValue V1, SDValue V2,

                                            SelectionDAG &DAG) {

  return lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG);

}


/// Lower VECTOR_SHUFFLE into XVILVH (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef<int> Mask,

                                          MVT VT, SDValue V1, SDValue V2,

                                          SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  unsigned HalfSize = Mask.size() / 2;

  unsigned LeftSize = HalfSize / 2;

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, HalfSize - LeftSize,

                              1) &&

      fitsRegularPattern<int>(Begin + HalfSize, 2, End, HalfSize + LeftSize, 1))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 2, End - HalfSize,

                                   Mask.size() + HalfSize - LeftSize, 1) &&

           fitsRegularPattern<int>(Begin + HalfSize, 2, End,

                                   Mask.size() + HalfSize + LeftSize, 1))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, HalfSize - LeftSize,

                              1) &&

      fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End, HalfSize + LeftSize,

                              1))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize,

                                   Mask.size() + HalfSize - LeftSize, 1) &&

           fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End,

                                   Mask.size() + HalfSize + LeftSize, 1))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VILVH, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into XVILVL (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef<int> Mask,

                                          MVT VT, SDValue V1, SDValue V2,

                                          SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &End = Mask.end();

  unsigned HalfSize = Mask.size() / 2;

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, 0, 1) &&

      fitsRegularPattern<int>(Begin + HalfSize, 2, End, HalfSize, 1))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, Mask.size(), 1) &&

           fitsRegularPattern<int>(Begin + HalfSize, 2, End,

                                   Mask.size() + HalfSize, 1))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, 0, 1) &&

      fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End, HalfSize, 1))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, Mask.size(),

                                   1) &&

           fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End,

                                   Mask.size() + HalfSize, 1))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VILVL, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into XVPICKEV (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef<int> Mask,

                                            MVT VT, SDValue V1, SDValue V2,

                                            SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &LeftMid = Mask.begin() + Mask.size() / 4;

  const auto &Mid = Mask.begin() + Mask.size() / 2;

  const auto &RightMid = Mask.end() - Mask.size() / 4;

  const auto &End = Mask.end();

  unsigned HalfSize = Mask.size() / 2;

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 1, LeftMid, 0, 2) &&

      fitsRegularPattern<int>(Mid, 1, RightMid, HalfSize, 2))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 1, LeftMid, Mask.size(), 2) &&

           fitsRegularPattern<int>(Mid, 1, RightMid, Mask.size() + HalfSize, 2))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(LeftMid, 1, Mid, 0, 2) &&

      fitsRegularPattern<int>(RightMid, 1, End, HalfSize, 2))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(LeftMid, 1, Mid, Mask.size(), 2) &&

           fitsRegularPattern<int>(RightMid, 1, End, Mask.size() + HalfSize, 2))

    V2 = OriV2;


  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VPICKEV, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into XVPICKOD (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef<int> Mask,

                                            MVT VT, SDValue V1, SDValue V2,

                                            SelectionDAG &DAG) {


  const auto &Begin = Mask.begin();

  const auto &LeftMid = Mask.begin() + Mask.size() / 4;

  const auto &Mid = Mask.begin() + Mask.size() / 2;

  const auto &RightMid = Mask.end() - Mask.size() / 4;

  const auto &End = Mask.end();

  unsigned HalfSize = Mask.size() / 2;

  SDValue OriV1 = V1, OriV2 = V2;


  if (fitsRegularPattern<int>(Begin, 1, LeftMid, 1, 2) &&

      fitsRegularPattern<int>(Mid, 1, RightMid, HalfSize + 1, 2))

    V1 = OriV1;

  else if (fitsRegularPattern<int>(Begin, 1, LeftMid, Mask.size() + 1, 2) &&

           fitsRegularPattern<int>(Mid, 1, RightMid, Mask.size() + HalfSize + 1,

                                   2))

    V1 = OriV2;

  else

    return SDValue();


  if (fitsRegularPattern<int>(LeftMid, 1, Mid, 1, 2) &&

      fitsRegularPattern<int>(RightMid, 1, End, HalfSize + 1, 2))

    V2 = OriV1;

  else if (fitsRegularPattern<int>(LeftMid, 1, Mid, Mask.size() + 1, 2) &&

           fitsRegularPattern<int>(RightMid, 1, End, Mask.size() + HalfSize + 1,

                                   2))

    V2 = OriV2;

  else

    return SDValue();


  return DAG.getNode(LoongArchISD::VPICKOD, DL, VT, V2, V1);

}


/// Lower VECTOR_SHUFFLE into XVSHUF (if possible).

static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef<int> Mask,

                                          MVT VT, SDValue V1, SDValue V2,

                                          SelectionDAG &DAG) {


  int MaskSize = Mask.size();

  int HalfSize = Mask.size() / 2;

  const auto &Begin = Mask.begin();

  const auto &Mid = Mask.begin() + HalfSize;

  const auto &End = Mask.end();


  // VECTOR_SHUFFLE concatenates the vectors:

  //  <0, 1, 2, 3, 4, 5, 6, 7> + <8, 9, 10, 11, 12, 13, 14, 15>

  //  shuffling ->

  //  <0, 1, 2, 3, 8, 9, 10, 11> <4, 5, 6, 7, 12, 13, 14, 15>

  //

  // XVSHUF concatenates the vectors:

  //  <a0, a1, a2, a3, b0, b1, b2, b3> + <a4, a5, a6, a7, b4, b5, b6, b7>

  //  shuffling ->

  //  <a0, a1, a2, a3, a4, a5, a6, a7> + <b0, b1, b2, b3, b4, b5, b6, b7>

  SmallVector<SDValue, 8> MaskAlloc;

  for (auto it = Begin; it < Mid; it++) {

    if (*it < 0) // UNDEF

      MaskAlloc.push_back(DAG.getTargetConstant(0, DL, MVT::i64));

    else if ((*it >= 0 && *it < HalfSize) ||

             (*it >= MaskSize && *it <= MaskSize + HalfSize)) {

      int M = *it < HalfSize ? *it : *it - HalfSize;

      MaskAlloc.push_back(DAG.getTargetConstant(M, DL, MVT::i64));

    } else

      return SDValue();

  }

  assert((int)MaskAlloc.size() == HalfSize && "xvshuf convert failed!");


  for (auto it = Mid; it < End; it++) {

    if (*it < 0) // UNDEF

      MaskAlloc.push_back(DAG.getTargetConstant(0, DL, MVT::i64));

    else if ((*it >= HalfSize && *it < MaskSize) ||

             (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {

      int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;

      MaskAlloc.push_back(DAG.getTargetConstant(M, DL, MVT::i64));

    } else

      return SDValue();

  }

  assert((int)MaskAlloc.size() == MaskSize && "xvshuf convert failed!");


  EVT MaskVecTy = VT.changeVectorElementTypeToInteger();

  SDValue MaskVec = DAG.getBuildVector(MaskVecTy, DL, MaskAlloc);

  return DAG.getNode(LoongArchISD::VSHUF, DL, VT, MaskVec, V2, V1);

}


/// Shuffle vectors by lane to generate more optimized instructions.

/// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.

///

/// Therefore, except for the following four cases, other cases are regarded

/// as cross-lane shuffles, where optimization is relatively limited.

///

/// - Shuffle high, low lanes of two inputs vector

///   <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>

/// - Shuffle low, high lanes of two inputs vector

///   <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>

/// - Shuffle low, low lanes of two inputs vector

///   <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>

/// - Shuffle high, high lanes of two inputs vector

///   <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>

///

/// The first case is the closest to LoongArch instructions and the other

/// cases need to be converted to it for processing.

///

/// This function may modify V1, V2 and Mask

static void canonicalizeShuffleVectorByLane(const SDLoc &DL,

                                            MutableArrayRef<int> Mask, MVT VT,

                                            SDValue &V1, SDValue &V2,

                                            SelectionDAG &DAG) {


  enum HalfMaskType { HighLaneTy, LowLaneTy, None };


  int MaskSize = Mask.size();

  int HalfSize = Mask.size() / 2;


  HalfMaskType preMask = None, postMask = None;


  if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](int M) {

        return M < 0 || (M >= 0 && M < HalfSize) ||

               (M >= MaskSize && M < MaskSize + HalfSize);

      }))

    preMask = HighLaneTy;

  else if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](int M) {

             return M < 0 || (M >= HalfSize && M < MaskSize) ||

                    (M >= MaskSize + HalfSize && M < MaskSize * 2);

           }))

    preMask = LowLaneTy;


  if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](int M) {

        return M < 0 || (M >= 0 && M < HalfSize) ||

               (M >= MaskSize && M < MaskSize + HalfSize);

      }))

    postMask = HighLaneTy;

  else if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](int M) {

             return M < 0 || (M >= HalfSize && M < MaskSize) ||

                    (M >= MaskSize + HalfSize && M < MaskSize * 2);

           }))

    postMask = LowLaneTy;


  // The pre-half of mask is high lane type, and the post-half of mask

  // is low lane type, which is closest to the LoongArch instructions.

  //

  // Note: In the LoongArch architecture, the high lane of mask corresponds

  // to the lower 128-bit of vector register, and the low lane of mask

  // corresponds the higher 128-bit of vector register.

  if (preMask == HighLaneTy && postMask == LowLaneTy) {

    return;

  }

  if (preMask == LowLaneTy && postMask == HighLaneTy) {

    V1 = DAG.getBitcast(MVT::v4i64, V1);

    V1 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V1,

                     DAG.getConstant(0b01001110, DL, MVT::i64));

    V1 = DAG.getBitcast(VT, V1);


    if (!V2.isUndef()) {

      V2 = DAG.getBitcast(MVT::v4i64, V2);

      V2 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V2,

                       DAG.getConstant(0b01001110, DL, MVT::i64));

      V2 = DAG.getBitcast(VT, V2);

    }


    for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {

      *it = *it < 0 ? *it : *it - HalfSize;

    }

    for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {

      *it = *it < 0 ? *it : *it + HalfSize;

    }

  } else if (preMask == LowLaneTy && postMask == LowLaneTy) {

    V1 = DAG.getBitcast(MVT::v4i64, V1);

    V1 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V1,

                     DAG.getConstant(0b11101110, DL, MVT::i64));

    V1 = DAG.getBitcast(VT, V1);


    if (!V2.isUndef()) {

      V2 = DAG.getBitcast(MVT::v4i64, V2);

      V2 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V2,

                       DAG.getConstant(0b11101110, DL, MVT::i64));

      V2 = DAG.getBitcast(VT, V2);

    }


    for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {

      *it = *it < 0 ? *it : *it - HalfSize;

    }

  } else if (preMask == HighLaneTy && postMask == HighLaneTy) {

    V1 = DAG.getBitcast(MVT::v4i64, V1);

    V1 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V1,

                     DAG.getConstant(0b01000100, DL, MVT::i64));

    V1 = DAG.getBitcast(VT, V1);


    if (!V2.isUndef()) {

      V2 = DAG.getBitcast(MVT::v4i64, V2);

      V2 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V2,

                       DAG.getConstant(0b01000100, DL, MVT::i64));

      V2 = DAG.getBitcast(VT, V2);

    }


    for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {

      *it = *it < 0 ? *it : *it + HalfSize;

    }

  } else { // cross-lane

    return;

  }

}


/// Dispatching routine to lower various 256-bit LoongArch vector shuffles.

///

/// This routine breaks down the specific type of 256-bit shuffle and

/// dispatches to the lowering routines accordingly.

static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,

                                  SDValue V1, SDValue V2, SelectionDAG &DAG) {

  assert((VT.SimpleTy == MVT::v32i8 || VT.SimpleTy == MVT::v16i16 ||

          VT.SimpleTy == MVT::v8i32 || VT.SimpleTy == MVT::v4i64 ||

          VT.SimpleTy == MVT::v8f32 || VT.SimpleTy == MVT::v4f64) &&

         "Vector type is unsupported for lasx!");

  assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&

         "Two operands have different types!");

  assert(VT.getVectorNumElements() == Mask.size() &&

         "Unexpected mask size for shuffle!");

  assert(Mask.size() % 2 == 0 && "Expected even mask size.");

  assert(Mask.size() >= 4 && "Mask size is less than 4.");


  // canonicalize non cross-lane shuffle vector

  SmallVector<int> NewMask(Mask);

  canonicalizeShuffleVectorByLane(DL, NewMask, VT, V1, V2, DAG);


  SDValue Result;

  // TODO: Add more comparison patterns.

  if (V2.isUndef()) {

    if ((Result = lowerVECTOR_SHUFFLE_XVREPLVEI(DL, NewMask, VT, V1, V2, DAG)))

      return Result;

    if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, NewMask, VT, V1, V2, DAG)))

      return Result;


    // TODO: This comment may be enabled in the future to better match the

    // pattern for instruction selection.

    /* V2 = V1; */

  }


  // It is recommended not to change the pattern comparison order for better

  // performance.

  if ((Result = lowerVECTOR_SHUFFLE_XVPACKEV(DL, NewMask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_XVPACKOD(DL, NewMask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_XVILVH(DL, NewMask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_XVILVL(DL, NewMask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_XVPICKEV(DL, NewMask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_XVPICKOD(DL, NewMask, VT, V1, V2, DAG)))

    return Result;

  if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, NewMask, VT, V1, V2, DAG)))

    return Result;


  return SDValue();

}


SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,

                                                     SelectionDAG &DAG) const {

  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);

  ArrayRef<int> OrigMask = SVOp->getMask();

  SDValue V1 = Op.getOperand(0);

  SDValue V2 = Op.getOperand(1);

  MVT VT = Op.getSimpleValueType();

  int NumElements = VT.getVectorNumElements();

  SDLoc DL(Op);


  bool V1IsUndef = V1.isUndef();

  bool V2IsUndef = V2.isUndef();

  if (V1IsUndef && V2IsUndef)

    return DAG.getUNDEF(VT);


  // When we create a shuffle node we put the UNDEF node to second operand,

  // but in some cases the first operand may be transformed to UNDEF.

  // In this case we should just commute the node.

  if (V1IsUndef)

    return DAG.getCommutedVectorShuffle(*SVOp);


  // Check for non-undef masks pointing at an undef vector and make the masks

  // undef as well. This makes it easier to match the shuffle based solely on

  // the mask.

  if (V2IsUndef &&

      any_of(OrigMask, [NumElements](int M) { return M >= NumElements; })) {

    SmallVector<int, 8> NewMask(OrigMask);

    for (int &M : NewMask)

      if (M >= NumElements)

        M = -1;

    return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);

  }


  // Check for illegal shuffle mask element index values.

  int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);

  (void)MaskUpperLimit;

  assert(llvm::all_of(OrigMask,

                      [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&

         "Out of bounds shuffle index");


  // For each vector width, delegate to a specialized lowering routine.

  if (VT.is128BitVector())

    return lower128BitShuffle(DL, OrigMask, VT, V1, V2, DAG);


  if (VT.is256BitVector())

    return lower256BitShuffle(DL, OrigMask, VT, V1, V2, DAG);


  return SDValue();

}


static bool isConstantOrUndef(const SDValue Op) {

  if (Op->isUndef())

    return true;

  if (isa<ConstantSDNode>(Op))

    return true;

  if (isa<ConstantFPSDNode>(Op))

    return true;

  return false;

}


static bool isConstantOrUndefBUILD_VECTOR(const BuildVectorSDNode *Op) {

  for (unsigned i = 0; i < Op->getNumOperands(); ++i)

    if (isConstantOrUndef(Op->getOperand(i)))

      return true;

  return false;

}


SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,

                                                   SelectionDAG &DAG) const {

  BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Op);

  EVT ResTy = Op->getValueType(0);

  SDLoc DL(Op);

  APInt SplatValue, SplatUndef;

  unsigned SplatBitSize;

  bool HasAnyUndefs;

  bool Is128Vec = ResTy.is128BitVector();

  bool Is256Vec = ResTy.is256BitVector();


  if ((!Subtarget.hasExtLSX() || !Is128Vec) &&

      (!Subtarget.hasExtLASX() || !Is256Vec))

    return SDValue();


  if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,

                            /*MinSplatBits=*/8) &&

      SplatBitSize <= 64) {

    // We can only cope with 8, 16, 32, or 64-bit elements.

    if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&

        SplatBitSize != 64)

      return SDValue();


    EVT ViaVecTy;


    switch (SplatBitSize) {

    default:

      return SDValue();

    case 8:

      ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;

      break;

    case 16:

      ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;

      break;

    case 32:

      ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;

      break;

    case 64:

      ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;

      break;

    }


    // SelectionDAG::getConstant will promote SplatValue appropriately.

    SDValue Result = DAG.getConstant(SplatValue, DL, ViaVecTy);


    // Bitcast to the type we originally wanted.

    if (ViaVecTy != ResTy)

      Result = DAG.getNode(ISD::BITCAST, SDLoc(Node), ResTy, Result);


    return Result;

  }


  if (DAG.isSplatValue(Op, /*AllowUndefs=*/false))

    return Op;


  if (!isConstantOrUndefBUILD_VECTOR(Node)) {

    // Use INSERT_VECTOR_ELT operations rather than expand to stores.

    // The resulting code is the same length as the expansion, but it doesn't

    // use memory operations.

    EVT ResTy = Node->getValueType(0);


    assert(ResTy.isVector());


    unsigned NumElts = ResTy.getVectorNumElements();

    SDValue Vector = DAG.getUNDEF(ResTy);

    for (unsigned i = 0; i < NumElts; ++i) {

      Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Vector,

                           Node->getOperand(i),

                           DAG.getConstant(i, DL, Subtarget.getGRLenVT()));

    }

    return Vector;

  }


  return SDValue();

}


SDValue

LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,

                                                 SelectionDAG &DAG) const {

  EVT VecTy = Op->getOperand(0)->getValueType(0);

  SDValue Idx = Op->getOperand(1);

  EVT EltTy = VecTy.getVectorElementType();

  unsigned NumElts = VecTy.getVectorNumElements();


  if (isa<ConstantSDNode>(Idx) &&

      (EltTy == MVT::i32 || EltTy == MVT::i64 || EltTy == MVT::f32 ||

       EltTy == MVT::f64 || Idx->getAsZExtVal() < NumElts / 2))

    return Op;


  return SDValue();

}


SDValue

LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,

                                                SelectionDAG &DAG) const {

  if (isa<ConstantSDNode>(Op->getOperand(2)))

    return Op;

  return SDValue();

}


SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,

                                                   SelectionDAG &DAG) const {

  SDLoc DL(Op);

  SyncScope::ID FenceSSID =

      static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));


  // singlethread fences only synchronize with signal handlers on the same

  // thread and thus only need to preserve instruction order, not actually

  // enforce memory ordering.

  if (FenceSSID == SyncScope::SingleThread)

    // MEMBARRIER is a compiler barrier; it codegens to a no-op.

    return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));


  return Op;

}


SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,

                                                     SelectionDAG &DAG) const {


  if (Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i32) {

    DAG.getContext()->emitError(

        "On LA64, only 64-bit registers can be written.");

    return Op.getOperand(0);

  }


  if (!Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i64) {

    DAG.getContext()->emitError(

        "On LA32, only 32-bit registers can be written.");

    return Op.getOperand(0);

  }


  return Op;

}


SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,

                                                SelectionDAG &DAG) const {

  if (!isa<ConstantSDNode>(Op.getOperand(0))) {

    DAG.getContext()->emitError("argument to '__builtin_frame_address' must "

                                "be a constant integer");

    return SDValue();

  }


  MachineFunction &MF = DAG.getMachineFunction();

  MF.getFrameInfo().setFrameAddressIsTaken(true);

  Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);

  EVT VT = Op.getValueType();

  SDLoc DL(Op);

  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);

  unsigned Depth = Op.getConstantOperandVal(0);

  int GRLenInBytes = Subtarget.getGRLen() / 8;


  while (Depth--) {

    int Offset = -(GRLenInBytes * 2);

    SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,

                              DAG.getSignedConstant(Offset, DL, VT));

    FrameAddr =

        DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());

  }

  return FrameAddr;

}


SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,

                                                 SelectionDAG &DAG) const {

  if (verifyReturnAddressArgumentIsConstant(Op, DAG))

    return SDValue();


  // Currently only support lowering return address for current frame.

  if (Op.getConstantOperandVal(0) != 0) {

    DAG.getContext()->emitError(

        "return address can only be determined for the current frame");

    return SDValue();

  }


  MachineFunction &MF = DAG.getMachineFunction();

  MF.getFrameInfo().setReturnAddressIsTaken(true);

  MVT GRLenVT = Subtarget.getGRLenVT();


  // Return the value of the return address register, marking it an implicit

  // live-in.

  Register Reg = MF.addLiveIn(Subtarget.getRegisterInfo()->getRARegister(),

                              getRegClassFor(GRLenVT));

  return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), Reg, GRLenVT);

}


SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op,

                                                   SelectionDAG &DAG) const {

  MachineFunction &MF = DAG.getMachineFunction();

  auto Size = Subtarget.getGRLen() / 8;

  auto FI = MF.getFrameInfo().CreateFixedObject(Size, 0, false);

  return DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));

}


SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op,

                                              SelectionDAG &DAG) const {

  MachineFunction &MF = DAG.getMachineFunction();

  auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>();


  SDLoc DL(Op);

  SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),

                                 getPointerTy(MF.getDataLayout()));


  // vastart just stores the address of the VarArgsFrameIndex slot into the

  // memory location argument.

  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();

  return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),

                      MachinePointerInfo(SV));

}


SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,

                                                 SelectionDAG &DAG) const {

  assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&

         !Subtarget.hasBasicD() && "unexpected target features");


  SDLoc DL(Op);

  SDValue Op0 = Op.getOperand(0);

  if (Op0->getOpcode() == ISD::AND) {

    auto *C = dyn_cast<ConstantSDNode>(Op0.getOperand(1));

    if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF))

      return Op;

  }


  if (Op0->getOpcode() == LoongArchISD::BSTRPICK &&

      Op0.getConstantOperandVal(1) < UINT64_C(0X1F) &&

      Op0.getConstantOperandVal(2) == UINT64_C(0))

    return Op;


  if (Op0.getOpcode() == ISD::AssertZext &&

      dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLT(MVT::i32))

    return Op;


  EVT OpVT = Op0.getValueType();

  EVT RetVT = Op.getValueType();

  RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);

  MakeLibCallOptions CallOptions;

  CallOptions.setTypeListBeforeSoften(OpVT, RetVT, true);

  SDValue Chain = SDValue();

  SDValue Result;

  std::tie(Result, Chain) =

      makeLibCall(DAG, LC, Op.getValueType(), Op0, CallOptions, DL, Chain);

  return Result;

}


SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,

                                                 SelectionDAG &DAG) const {

  assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&

         !Subtarget.hasBasicD() && "unexpected target features");


  SDLoc DL(Op);

  SDValue Op0 = Op.getOperand(0);


  if ((Op0.getOpcode() == ISD::AssertSext ||

       Op0.getOpcode() == ISD::SIGN_EXTEND_INREG) &&

      dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLE(MVT::i32))

    return Op;


  EVT OpVT = Op0.getValueType();

  EVT RetVT = Op.getValueType();

  RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);

  MakeLibCallOptions CallOptions;

  CallOptions.setTypeListBeforeSoften(OpVT, RetVT, true);

  SDValue Chain = SDValue();

  SDValue Result;

  std::tie(Result, Chain) =

      makeLibCall(DAG, LC, Op.getValueType(), Op0, CallOptions, DL, Chain);

  return Result;

}


SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,

                                              SelectionDAG &DAG) const {


  SDLoc DL(Op);

  SDValue Op0 = Op.getOperand(0);


  if (Op.getValueType() == MVT::f32 && Op0.getValueType() == MVT::i32 &&

      Subtarget.is64Bit() && Subtarget.hasBasicF()) {

    SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);

    return DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, NewOp0);

  }

  return Op;

}


SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,

                                                 SelectionDAG &DAG) const {


  SDLoc DL(Op);

  SDValue Op0 = Op.getOperand(0);


  if (Op0.getValueType() == MVT::f16)

    Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);


  if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&

      !Subtarget.hasBasicD()) {

    SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, MVT::f32, Op0);

    return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Dst);

  }


  EVT FPTy = EVT::getFloatingPointVT(Op.getValueSizeInBits());

  SDValue Trunc = DAG.getNode(LoongArchISD::FTINT, DL, FPTy, Op0);

  return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Trunc);

}


static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,

                             SelectionDAG &DAG, unsigned Flags) {

  return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);

}


static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,

                             SelectionDAG &DAG, unsigned Flags) {

  return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),

                                   Flags);

}


static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,

                             SelectionDAG &DAG, unsigned Flags) {

  return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),

                                   N->getOffset(), Flags);

}


static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,

                             SelectionDAG &DAG, unsigned Flags) {

  return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);

}


template <class NodeTy>

SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,

                                         CodeModel::Model M,

                                         bool IsLocal) const {

  SDLoc DL(N);

  EVT Ty = getPointerTy(DAG.getDataLayout());

  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);

  SDValue Load;


  switch (M) {

  default:

    report_fatal_error("Unsupported code model");


  case CodeModel::Large: {

    assert(Subtarget.is64Bit() && "Large code model requires LA64");


    // This is not actually used, but is necessary for successfully matching

    // the PseudoLA_*_LARGE nodes.

    SDValue Tmp = DAG.getConstant(0, DL, Ty);

    if (IsLocal) {

      // This generates the pattern (PseudoLA_PCREL_LARGE tmp sym), that

      // eventually becomes the desired 5-insn code sequence.

      Load = SDValue(DAG.getMachineNode(LoongArch::PseudoLA_PCREL_LARGE, DL, Ty,

                                        Tmp, Addr),

                     0);

    } else {

      // This generates the pattern (PseudoLA_GOT_LARGE tmp sym), that

      // eventually becomes the desired 5-insn code sequence.

      Load = SDValue(

          DAG.getMachineNode(LoongArch::PseudoLA_GOT_LARGE, DL, Ty, Tmp, Addr),

          0);

    }

    break;

  }


  case CodeModel::Small:

  case CodeModel::Medium:

    if (IsLocal) {

      // This generates the pattern (PseudoLA_PCREL sym), which expands to

      // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)).

      Load = SDValue(

          DAG.getMachineNode(LoongArch::PseudoLA_PCREL, DL, Ty, Addr), 0);

    } else {

      // This generates the pattern (PseudoLA_GOT sym), which expands to (ld.w/d

      // (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)).

      Load =

          SDValue(DAG.getMachineNode(LoongArch::PseudoLA_GOT, DL, Ty, Addr), 0);

    }

  }


  if (!IsLocal) {

    // Mark the load instruction as invariant to enable hoisting in MachineLICM.

    MachineFunction &MF = DAG.getMachineFunction();

    MachineMemOperand *MemOp = MF.getMachineMemOperand(

        MachinePointerInfo::getGOT(MF),

        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |

            MachineMemOperand::MOInvariant,

        LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));

    DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});

  }


  return Load;

}


SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op,

                                                   SelectionDAG &DAG) const {

  return getAddr(cast<BlockAddressSDNode>(Op), DAG,

                 DAG.getTarget().getCodeModel());

}


SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op,

                                                SelectionDAG &DAG) const {

  return getAddr(cast<JumpTableSDNode>(Op), DAG,

                 DAG.getTarget().getCodeModel());

}


SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op,

                                                   SelectionDAG &DAG) const {

  return getAddr(cast<ConstantPoolSDNode>(Op), DAG,

                 DAG.getTarget().getCodeModel());

}


SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op,

                                                    SelectionDAG &DAG) const {

  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);

  assert(N->getOffset() == 0 && "unexpected offset in global node");

  auto CM = DAG.getTarget().getCodeModel();

  const GlobalValue *GV = N->getGlobal();


  if (GV->isDSOLocal() && isa<GlobalVariable>(GV)) {

    if (auto GCM = dyn_cast<GlobalVariable>(GV)->getCodeModel())

      CM = *GCM;

  }


  return getAddr(N, DAG, CM, GV->isDSOLocal());

}


SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,

                                                  SelectionDAG &DAG,

                                                  unsigned Opc, bool UseGOT,

                                                  bool Large) const {

  SDLoc DL(N);

  EVT Ty = getPointerTy(DAG.getDataLayout());

  MVT GRLenVT = Subtarget.getGRLenVT();


  // This is not actually used, but is necessary for successfully matching the

  // PseudoLA_*_LARGE nodes.

  SDValue Tmp = DAG.getConstant(0, DL, Ty);

  SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0);

  SDValue Offset = Large

                       ? SDValue(DAG.getMachineNode(Opc, DL, Ty, Tmp, Addr), 0)

                       : SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0);

  if (UseGOT) {

    // Mark the load instruction as invariant to enable hoisting in MachineLICM.

    MachineFunction &MF = DAG.getMachineFunction();

    MachineMemOperand *MemOp = MF.getMachineMemOperand(

        MachinePointerInfo::getGOT(MF),

        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |

            MachineMemOperand::MOInvariant,

        LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));

    DAG.setNodeMemRefs(cast<MachineSDNode>(Offset.getNode()), {MemOp});

  }


  // Add the thread pointer.

  return DAG.getNode(ISD::ADD, DL, Ty, Offset,

                     DAG.getRegister(LoongArch::R2, GRLenVT));

}


SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,

                                                   SelectionDAG &DAG,

                                                   unsigned Opc,

                                                   bool Large) const {

  SDLoc DL(N);

  EVT Ty = getPointerTy(DAG.getDataLayout());

  IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());


  // This is not actually used, but is necessary for successfully matching the

  // PseudoLA_*_LARGE nodes.

  SDValue Tmp = DAG.getConstant(0, DL, Ty);


  // Use a PC-relative addressing mode to access the dynamic GOT address.

  SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0);

  SDValue Load = Large ? SDValue(DAG.getMachineNode(Opc, DL, Ty, Tmp, Addr), 0)

                       : SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0);


  // Prepare argument list to generate call.

  ArgListTy Args;

  ArgListEntry Entry;

  Entry.Node = Load;

  Entry.Ty = CallTy;

  Args.push_back(Entry);


  // Setup call to __tls_get_addr.

  TargetLowering::CallLoweringInfo CLI(DAG);

  CLI.setDebugLoc(DL)

      .setChain(DAG.getEntryNode())

      .setLibCallee(CallingConv::C, CallTy,

                    DAG.getExternalSymbol("__tls_get_addr", Ty),

                    std::move(Args));


  return LowerCallTo(CLI).first;

}


SDValue LoongArchTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,

                                                SelectionDAG &DAG, unsigned Opc,

                                                bool Large) const {

  SDLoc DL(N);

  EVT Ty = getPointerTy(DAG.getDataLayout());

  const GlobalValue *GV = N->getGlobal();


  // This is not actually used, but is necessary for successfully matching the

  // PseudoLA_*_LARGE nodes.

  SDValue Tmp = DAG.getConstant(0, DL, Ty);


  // Use a PC-relative addressing mode to access the global dynamic GOT address.

  // This generates the pattern (PseudoLA_TLS_DESC_PC{,LARGE} sym).

  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);

  return Large ? SDValue(DAG.getMachineNode(Opc, DL, Ty, Tmp, Addr), 0)

               : SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0);

}


SDValue

LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op,

                                               SelectionDAG &DAG) const {

  if (DAG.getMachineFunction().getFunction().getCallingConv() ==

      CallingConv::GHC)

    report_fatal_error("In GHC calling convention TLS is not supported");


  bool Large = DAG.getTarget().getCodeModel() == CodeModel::Large;

  assert((!Large || Subtarget.is64Bit()) && "Large code model requires LA64");


  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);

  assert(N->getOffset() == 0 && "unexpected offset in global node");


  if (DAG.getTarget().useEmulatedTLS())

    report_fatal_error("the emulated TLS is prohibited",

                       /*GenCrashDiag=*/false);


  bool IsDesc = DAG.getTarget().useTLSDESC();


  switch (getTargetMachine().getTLSModel(N->getGlobal())) {

  case TLSModel::GeneralDynamic:

    // In this model, application code calls the dynamic linker function

    // __tls_get_addr to locate TLS offsets into the dynamic thread vector at

    // runtime.

    if (!IsDesc)

      return getDynamicTLSAddr(N, DAG,

                               Large ? LoongArch::PseudoLA_TLS_GD_LARGE

                                     : LoongArch::PseudoLA_TLS_GD,

                               Large);

    break;

  case TLSModel::LocalDynamic:

    // Same as GeneralDynamic, except for assembly modifiers and relocation

    // records.

    if (!IsDesc)

      return getDynamicTLSAddr(N, DAG,

                               Large ? LoongArch::PseudoLA_TLS_LD_LARGE

                                     : LoongArch::PseudoLA_TLS_LD,

                               Large);

    break;

  case TLSModel::InitialExec:

    // This model uses the GOT to resolve TLS offsets.

    return getStaticTLSAddr(N, DAG,

                            Large ? LoongArch::PseudoLA_TLS_IE_LARGE

                                  : LoongArch::PseudoLA_TLS_IE,

                            /*UseGOT=*/true, Large);

  case TLSModel::LocalExec:

    // This model is used when static linking as the TLS offsets are resolved

    // during program linking.

    //

    // This node doesn't need an extra argument for the large code model.

    return getStaticTLSAddr(N, DAG, LoongArch::PseudoLA_TLS_LE,

                            /*UseGOT=*/false);

  }


  return getTLSDescAddr(N, DAG,

                        Large ? LoongArch::PseudoLA_TLS_DESC_LARGE

                              : LoongArch::PseudoLA_TLS_DESC,

                        Large);

}


template <unsigned N>

static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp,

                                    SelectionDAG &DAG, bool IsSigned = false) {

  auto *CImm = cast<ConstantSDNode>(Op->getOperand(ImmOp));

  // Check the ImmArg.

  if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||

      (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {

    DAG.getContext()->emitError(Op->getOperationName(0) +

                                ": argument out of range.");

    return DAG.getNode(ISD::UNDEF, SDLoc(Op), Op.getValueType());

  }

  return SDValue();

}


SDValue

LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,

                                                 SelectionDAG &DAG) const {

  SDLoc DL(Op);

  switch (Op.getConstantOperandVal(0)) {

  default:

    return SDValue(); // Don't custom lower most intrinsics.

  case Intrinsic::thread_pointer: {

    EVT PtrVT = getPointerTy(DAG.getDataLayout());

    return DAG.getRegister(LoongArch::R2, PtrVT);

  }

  case Intrinsic::loongarch_lsx_vpickve2gr_d:

  case Intrinsic::loongarch_lsx_vpickve2gr_du:

  case Intrinsic::loongarch_lsx_vreplvei_d:

  case Intrinsic::loongarch_lasx_xvrepl128vei_d:

    return checkIntrinsicImmArg<1>(Op, 2, DAG);

  case Intrinsic::loongarch_lsx_vreplvei_w:

  case Intrinsic::loongarch_lasx_xvrepl128vei_w:

  case Intrinsic::loongarch_lasx_xvpickve2gr_d:

  case Intrinsic::loongarch_lasx_xvpickve2gr_du:

  case Intrinsic::loongarch_lasx_xvpickve_d:

  case Intrinsic::loongarch_lasx_xvpickve_d_f:

    return checkIntrinsicImmArg<2>(Op, 2, DAG);

  case Intrinsic::loongarch_lasx_xvinsve0_d:

    return checkIntrinsicImmArg<2>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vsat_b:

  case Intrinsic::loongarch_lsx_vsat_bu:

  case Intrinsic::loongarch_lsx_vrotri_b:

  case Intrinsic::loongarch_lsx_vsllwil_h_b:

  case Intrinsic::loongarch_lsx_vsllwil_hu_bu:

  case Intrinsic::loongarch_lsx_vsrlri_b:

  case Intrinsic::loongarch_lsx_vsrari_b:

  case Intrinsic::loongarch_lsx_vreplvei_h:

  case Intrinsic::loongarch_lasx_xvsat_b:

  case Intrinsic::loongarch_lasx_xvsat_bu:

  case Intrinsic::loongarch_lasx_xvrotri_b:

  case Intrinsic::loongarch_lasx_xvsllwil_h_b:

  case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:

  case Intrinsic::loongarch_lasx_xvsrlri_b:

  case Intrinsic::loongarch_lasx_xvsrari_b:

  case Intrinsic::loongarch_lasx_xvrepl128vei_h:

  case Intrinsic::loongarch_lasx_xvpickve_w:

  case Intrinsic::loongarch_lasx_xvpickve_w_f:

    return checkIntrinsicImmArg<3>(Op, 2, DAG);

  case Intrinsic::loongarch_lasx_xvinsve0_w:

    return checkIntrinsicImmArg<3>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vsat_h:

  case Intrinsic::loongarch_lsx_vsat_hu:

  case Intrinsic::loongarch_lsx_vrotri_h:

  case Intrinsic::loongarch_lsx_vsllwil_w_h:

  case Intrinsic::loongarch_lsx_vsllwil_wu_hu:

  case Intrinsic::loongarch_lsx_vsrlri_h:

  case Intrinsic::loongarch_lsx_vsrari_h:

  case Intrinsic::loongarch_lsx_vreplvei_b:

  case Intrinsic::loongarch_lasx_xvsat_h:

  case Intrinsic::loongarch_lasx_xvsat_hu:

  case Intrinsic::loongarch_lasx_xvrotri_h:

  case Intrinsic::loongarch_lasx_xvsllwil_w_h:

  case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:

  case Intrinsic::loongarch_lasx_xvsrlri_h:

  case Intrinsic::loongarch_lasx_xvsrari_h:

  case Intrinsic::loongarch_lasx_xvrepl128vei_b:

    return checkIntrinsicImmArg<4>(Op, 2, DAG);

  case Intrinsic::loongarch_lsx_vsrlni_b_h:

  case Intrinsic::loongarch_lsx_vsrani_b_h:

  case Intrinsic::loongarch_lsx_vsrlrni_b_h:

  case Intrinsic::loongarch_lsx_vsrarni_b_h:

  case Intrinsic::loongarch_lsx_vssrlni_b_h:

  case Intrinsic::loongarch_lsx_vssrani_b_h:

  case Intrinsic::loongarch_lsx_vssrlni_bu_h:

  case Intrinsic::loongarch_lsx_vssrani_bu_h:

  case Intrinsic::loongarch_lsx_vssrlrni_b_h:

  case Intrinsic::loongarch_lsx_vssrarni_b_h:

  case Intrinsic::loongarch_lsx_vssrlrni_bu_h:

  case Intrinsic::loongarch_lsx_vssrarni_bu_h:

  case Intrinsic::loongarch_lasx_xvsrlni_b_h:

  case Intrinsic::loongarch_lasx_xvsrani_b_h:

  case Intrinsic::loongarch_lasx_xvsrlrni_b_h:

  case Intrinsic::loongarch_lasx_xvsrarni_b_h:

  case Intrinsic::loongarch_lasx_xvssrlni_b_h:

  case Intrinsic::loongarch_lasx_xvssrani_b_h:

  case Intrinsic::loongarch_lasx_xvssrlni_bu_h:

  case Intrinsic::loongarch_lasx_xvssrani_bu_h:

  case Intrinsic::loongarch_lasx_xvssrlrni_b_h:

  case Intrinsic::loongarch_lasx_xvssrarni_b_h:

  case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:

  case Intrinsic::loongarch_lasx_xvssrarni_bu_h:

    return checkIntrinsicImmArg<4>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vsat_w:

  case Intrinsic::loongarch_lsx_vsat_wu:

  case Intrinsic::loongarch_lsx_vrotri_w:

  case Intrinsic::loongarch_lsx_vsllwil_d_w:

  case Intrinsic::loongarch_lsx_vsllwil_du_wu:

  case Intrinsic::loongarch_lsx_vsrlri_w:

  case Intrinsic::loongarch_lsx_vsrari_w:

  case Intrinsic::loongarch_lsx_vslei_bu:

  case Intrinsic::loongarch_lsx_vslei_hu:

  case Intrinsic::loongarch_lsx_vslei_wu:

  case Intrinsic::loongarch_lsx_vslei_du:

  case Intrinsic::loongarch_lsx_vslti_bu:

  case Intrinsic::loongarch_lsx_vslti_hu:

  case Intrinsic::loongarch_lsx_vslti_wu:

  case Intrinsic::loongarch_lsx_vslti_du:

  case Intrinsic::loongarch_lsx_vbsll_v:

  case Intrinsic::loongarch_lsx_vbsrl_v:

  case Intrinsic::loongarch_lasx_xvsat_w:

  case Intrinsic::loongarch_lasx_xvsat_wu:

  case Intrinsic::loongarch_lasx_xvrotri_w:

  case Intrinsic::loongarch_lasx_xvsllwil_d_w:

  case Intrinsic::loongarch_lasx_xvsllwil_du_wu:

  case Intrinsic::loongarch_lasx_xvsrlri_w:

  case Intrinsic::loongarch_lasx_xvsrari_w:

  case Intrinsic::loongarch_lasx_xvslei_bu:

  case Intrinsic::loongarch_lasx_xvslei_hu:

  case Intrinsic::loongarch_lasx_xvslei_wu:

  case Intrinsic::loongarch_lasx_xvslei_du:

  case Intrinsic::loongarch_lasx_xvslti_bu:

  case Intrinsic::loongarch_lasx_xvslti_hu:

  case Intrinsic::loongarch_lasx_xvslti_wu:

  case Intrinsic::loongarch_lasx_xvslti_du:

  case Intrinsic::loongarch_lasx_xvbsll_v:

  case Intrinsic::loongarch_lasx_xvbsrl_v:

    return checkIntrinsicImmArg<5>(Op, 2, DAG);

  case Intrinsic::loongarch_lsx_vseqi_b:

  case Intrinsic::loongarch_lsx_vseqi_h:

  case Intrinsic::loongarch_lsx_vseqi_w:

  case Intrinsic::loongarch_lsx_vseqi_d:

  case Intrinsic::loongarch_lsx_vslei_b:

  case Intrinsic::loongarch_lsx_vslei_h:

  case Intrinsic::loongarch_lsx_vslei_w:

  case Intrinsic::loongarch_lsx_vslei_d:

  case Intrinsic::loongarch_lsx_vslti_b:

  case Intrinsic::loongarch_lsx_vslti_h:

  case Intrinsic::loongarch_lsx_vslti_w:

  case Intrinsic::loongarch_lsx_vslti_d:

  case Intrinsic::loongarch_lasx_xvseqi_b:

  case Intrinsic::loongarch_lasx_xvseqi_h:

  case Intrinsic::loongarch_lasx_xvseqi_w:

  case Intrinsic::loongarch_lasx_xvseqi_d:

  case Intrinsic::loongarch_lasx_xvslei_b:

  case Intrinsic::loongarch_lasx_xvslei_h:

  case Intrinsic::loongarch_lasx_xvslei_w:

  case Intrinsic::loongarch_lasx_xvslei_d:

  case Intrinsic::loongarch_lasx_xvslti_b:

  case Intrinsic::loongarch_lasx_xvslti_h:

  case Intrinsic::loongarch_lasx_xvslti_w:

  case Intrinsic::loongarch_lasx_xvslti_d:

    return checkIntrinsicImmArg<5>(Op, 2, DAG, /*IsSigned=*/true);

  case Intrinsic::loongarch_lsx_vsrlni_h_w:

  case Intrinsic::loongarch_lsx_vsrani_h_w:

  case Intrinsic::loongarch_lsx_vsrlrni_h_w:

  case Intrinsic::loongarch_lsx_vsrarni_h_w:

  case Intrinsic::loongarch_lsx_vssrlni_h_w:

  case Intrinsic::loongarch_lsx_vssrani_h_w:

  case Intrinsic::loongarch_lsx_vssrlni_hu_w:

  case Intrinsic::loongarch_lsx_vssrani_hu_w:

  case Intrinsic::loongarch_lsx_vssrlrni_h_w:

  case Intrinsic::loongarch_lsx_vssrarni_h_w:

  case Intrinsic::loongarch_lsx_vssrlrni_hu_w:

  case Intrinsic::loongarch_lsx_vssrarni_hu_w:

  case Intrinsic::loongarch_lsx_vfrstpi_b:

  case Intrinsic::loongarch_lsx_vfrstpi_h:

  case Intrinsic::loongarch_lasx_xvsrlni_h_w:

  case Intrinsic::loongarch_lasx_xvsrani_h_w:

  case Intrinsic::loongarch_lasx_xvsrlrni_h_w:

  case Intrinsic::loongarch_lasx_xvsrarni_h_w:

  case Intrinsic::loongarch_lasx_xvssrlni_h_w:

  case Intrinsic::loongarch_lasx_xvssrani_h_w:

  case Intrinsic::loongarch_lasx_xvssrlni_hu_w:

  case Intrinsic::loongarch_lasx_xvssrani_hu_w:

  case Intrinsic::loongarch_lasx_xvssrlrni_h_w:

  case Intrinsic::loongarch_lasx_xvssrarni_h_w:

  case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:

  case Intrinsic::loongarch_lasx_xvssrarni_hu_w:

  case Intrinsic::loongarch_lasx_xvfrstpi_b:

  case Intrinsic::loongarch_lasx_xvfrstpi_h:

    return checkIntrinsicImmArg<5>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vsat_d:

  case Intrinsic::loongarch_lsx_vsat_du:

  case Intrinsic::loongarch_lsx_vrotri_d:

  case Intrinsic::loongarch_lsx_vsrlri_d:

  case Intrinsic::loongarch_lsx_vsrari_d:

  case Intrinsic::loongarch_lasx_xvsat_d:

  case Intrinsic::loongarch_lasx_xvsat_du:

  case Intrinsic::loongarch_lasx_xvrotri_d:

  case Intrinsic::loongarch_lasx_xvsrlri_d:

  case Intrinsic::loongarch_lasx_xvsrari_d:

    return checkIntrinsicImmArg<6>(Op, 2, DAG);

  case Intrinsic::loongarch_lsx_vsrlni_w_d:

  case Intrinsic::loongarch_lsx_vsrani_w_d:

  case Intrinsic::loongarch_lsx_vsrlrni_w_d:

  case Intrinsic::loongarch_lsx_vsrarni_w_d:

  case Intrinsic::loongarch_lsx_vssrlni_w_d:

  case Intrinsic::loongarch_lsx_vssrani_w_d:

  case Intrinsic::loongarch_lsx_vssrlni_wu_d:

  case Intrinsic::loongarch_lsx_vssrani_wu_d:

  case Intrinsic::loongarch_lsx_vssrlrni_w_d:

  case Intrinsic::loongarch_lsx_vssrarni_w_d:

  case Intrinsic::loongarch_lsx_vssrlrni_wu_d:

  case Intrinsic::loongarch_lsx_vssrarni_wu_d:

  case Intrinsic::loongarch_lasx_xvsrlni_w_d:

  case Intrinsic::loongarch_lasx_xvsrani_w_d:

  case Intrinsic::loongarch_lasx_xvsrlrni_w_d:

  case Intrinsic::loongarch_lasx_xvsrarni_w_d:

  case Intrinsic::loongarch_lasx_xvssrlni_w_d:

  case Intrinsic::loongarch_lasx_xvssrani_w_d:

  case Intrinsic::loongarch_lasx_xvssrlni_wu_d:

  case Intrinsic::loongarch_lasx_xvssrani_wu_d:

  case Intrinsic::loongarch_lasx_xvssrlrni_w_d:

  case Intrinsic::loongarch_lasx_xvssrarni_w_d:

  case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:

  case Intrinsic::loongarch_lasx_xvssrarni_wu_d:

    return checkIntrinsicImmArg<6>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vsrlni_d_q:

  case Intrinsic::loongarch_lsx_vsrani_d_q:

  case Intrinsic::loongarch_lsx_vsrlrni_d_q:

  case Intrinsic::loongarch_lsx_vsrarni_d_q:

  case Intrinsic::loongarch_lsx_vssrlni_d_q:

  case Intrinsic::loongarch_lsx_vssrani_d_q:

  case Intrinsic::loongarch_lsx_vssrlni_du_q:

  case Intrinsic::loongarch_lsx_vssrani_du_q:

  case Intrinsic::loongarch_lsx_vssrlrni_d_q:

  case Intrinsic::loongarch_lsx_vssrarni_d_q:

  case Intrinsic::loongarch_lsx_vssrlrni_du_q:

  case Intrinsic::loongarch_lsx_vssrarni_du_q:

  case Intrinsic::loongarch_lasx_xvsrlni_d_q:

  case Intrinsic::loongarch_lasx_xvsrani_d_q:

  case Intrinsic::loongarch_lasx_xvsrlrni_d_q:

  case Intrinsic::loongarch_lasx_xvsrarni_d_q:

  case Intrinsic::loongarch_lasx_xvssrlni_d_q:

  case Intrinsic::loongarch_lasx_xvssrani_d_q:

  case Intrinsic::loongarch_lasx_xvssrlni_du_q:

  case Intrinsic::loongarch_lasx_xvssrani_du_q:

  case Intrinsic::loongarch_lasx_xvssrlrni_d_q:

  case Intrinsic::loongarch_lasx_xvssrarni_d_q:

  case Intrinsic::loongarch_lasx_xvssrlrni_du_q:

  case Intrinsic::loongarch_lasx_xvssrarni_du_q:

    return checkIntrinsicImmArg<7>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vnori_b:

  case Intrinsic::loongarch_lsx_vshuf4i_b:

  case Intrinsic::loongarch_lsx_vshuf4i_h:

  case Intrinsic::loongarch_lsx_vshuf4i_w:

  case Intrinsic::loongarch_lasx_xvnori_b:

  case Intrinsic::loongarch_lasx_xvshuf4i_b:

  case Intrinsic::loongarch_lasx_xvshuf4i_h:

  case Intrinsic::loongarch_lasx_xvshuf4i_w:

  case Intrinsic::loongarch_lasx_xvpermi_d:

    return checkIntrinsicImmArg<8>(Op, 2, DAG);

  case Intrinsic::loongarch_lsx_vshuf4i_d:

  case Intrinsic::loongarch_lsx_vpermi_w:

  case Intrinsic::loongarch_lsx_vbitseli_b:

  case Intrinsic::loongarch_lsx_vextrins_b:

  case Intrinsic::loongarch_lsx_vextrins_h:

  case Intrinsic::loongarch_lsx_vextrins_w:

  case Intrinsic::loongarch_lsx_vextrins_d:

  case Intrinsic::loongarch_lasx_xvshuf4i_d:

  case Intrinsic::loongarch_lasx_xvpermi_w:

  case Intrinsic::loongarch_lasx_xvpermi_q:

  case Intrinsic::loongarch_lasx_xvbitseli_b:

  case Intrinsic::loongarch_lasx_xvextrins_b:

  case Intrinsic::loongarch_lasx_xvextrins_h:

  case Intrinsic::loongarch_lasx_xvextrins_w:

  case Intrinsic::loongarch_lasx_xvextrins_d:

    return checkIntrinsicImmArg<8>(Op, 3, DAG);

  case Intrinsic::loongarch_lsx_vrepli_b:

  case Intrinsic::loongarch_lsx_vrepli_h:

  case Intrinsic::loongarch_lsx_vrepli_w:

  case Intrinsic::loongarch_lsx_vrepli_d:

  case Intrinsic::loongarch_lasx_xvrepli_b:

  case Intrinsic::loongarch_lasx_xvrepli_h:

  case Intrinsic::loongarch_lasx_xvrepli_w:

  case Intrinsic::loongarch_lasx_xvrepli_d:

    return checkIntrinsicImmArg<10>(Op, 1, DAG, /*IsSigned=*/true);

  case Intrinsic::loongarch_lsx_vldi:

  case Intrinsic::loongarch_lasx_xvldi:

    return checkIntrinsicImmArg<13>(Op, 1, DAG, /*IsSigned=*/true);

  }

}


// Helper function that emits error message for intrinsics with chain and return

// merge values of a UNDEF and the chain.

static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op,

                                                  StringRef ErrorMsg,

                                                  SelectionDAG &DAG) {

  DAG.getContext()->emitError(Op->getOperationName(0) + ": " + ErrorMsg + ".");

  return DAG.getMergeValues({DAG.getUNDEF(Op.getValueType()), Op.getOperand(0)},

                            SDLoc(Op));

}


SDValue

LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,

                                                SelectionDAG &DAG) const {

  SDLoc DL(Op);

  MVT GRLenVT = Subtarget.getGRLenVT();

  EVT VT = Op.getValueType();

  SDValue Chain = Op.getOperand(0);

  const StringRef ErrorMsgOOR = "argument out of range";

  const StringRef ErrorMsgReqLA64 = "requires loongarch64";

  const StringRef ErrorMsgReqF = "requires basic 'f' target feature";


  switch (Op.getConstantOperandVal(1)) {

  default:

    return Op;

  case Intrinsic::loongarch_crc_w_b_w:

  case Intrinsic::loongarch_crc_w_h_w:

  case Intrinsic::loongarch_crc_w_w_w:

  case Intrinsic::loongarch_crc_w_d_w:

  case Intrinsic::loongarch_crcc_w_b_w:

  case Intrinsic::loongarch_crcc_w_h_w:

  case Intrinsic::loongarch_crcc_w_w_w:

  case Intrinsic::loongarch_crcc_w_d_w:

    return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgReqLA64, DAG);

  case Intrinsic::loongarch_csrrd_w:

  case Intrinsic::loongarch_csrrd_d: {

    unsigned Imm = Op.getConstantOperandVal(2);

    return !isUInt<14>(Imm)

               ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::CSRRD, DL, {GRLenVT, MVT::Other},

                             {Chain, DAG.getConstant(Imm, DL, GRLenVT)});

  }

  case Intrinsic::loongarch_csrwr_w:

  case Intrinsic::loongarch_csrwr_d: {

    unsigned Imm = Op.getConstantOperandVal(3);

    return !isUInt<14>(Imm)

               ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::CSRWR, DL, {GRLenVT, MVT::Other},

                             {Chain, Op.getOperand(2),

                              DAG.getConstant(Imm, DL, GRLenVT)});

  }

  case Intrinsic::loongarch_csrxchg_w:

  case Intrinsic::loongarch_csrxchg_d: {

    unsigned Imm = Op.getConstantOperandVal(4);

    return !isUInt<14>(Imm)

               ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::CSRXCHG, DL, {GRLenVT, MVT::Other},

                             {Chain, Op.getOperand(2), Op.getOperand(3),

                              DAG.getConstant(Imm, DL, GRLenVT)});

  }

  case Intrinsic::loongarch_iocsrrd_d: {

    return DAG.getNode(

        LoongArchISD::IOCSRRD_D, DL, {GRLenVT, MVT::Other},

        {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2))});

  }

#define IOCSRRD_CASE(NAME, NODE)                                               \

  case Intrinsic::loongarch_##NAME: {                                          \

    return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other},          \

                       {Chain, Op.getOperand(2)});                             \

  }

    IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);

    IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);

    IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);

#undef IOCSRRD_CASE

  case Intrinsic::loongarch_cpucfg: {

    return DAG.getNode(LoongArchISD::CPUCFG, DL, {GRLenVT, MVT::Other},

                       {Chain, Op.getOperand(2)});

  }

  case Intrinsic::loongarch_lddir_d: {

    unsigned Imm = Op.getConstantOperandVal(3);

    return !isUInt<8>(Imm)

               ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)

               : Op;

  }

  case Intrinsic::loongarch_movfcsr2gr: {

    if (!Subtarget.hasBasicF())

      return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgReqF, DAG);

    unsigned Imm = Op.getConstantOperandVal(2);

    return !isUInt<2>(Imm)

               ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::MOVFCSR2GR, DL, {VT, MVT::Other},

                             {Chain, DAG.getConstant(Imm, DL, GRLenVT)});

  }

  case Intrinsic::loongarch_lsx_vld:

  case Intrinsic::loongarch_lsx_vldrepl_b:

  case Intrinsic::loongarch_lasx_xvld:

  case Intrinsic::loongarch_lasx_xvldrepl_b:

    return !isInt<12>(cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())

               ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vldrepl_h:

  case Intrinsic::loongarch_lasx_xvldrepl_h:

    return !isShiftedInt<11, 1>(

               cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())

               ? emitIntrinsicWithChainErrorMessage(

                     Op, "argument out of range or not a multiple of 2", DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vldrepl_w:

  case Intrinsic::loongarch_lasx_xvldrepl_w:

    return !isShiftedInt<10, 2>(

               cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())

               ? emitIntrinsicWithChainErrorMessage(

                     Op, "argument out of range or not a multiple of 4", DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vldrepl_d:

  case Intrinsic::loongarch_lasx_xvldrepl_d:

    return !isShiftedInt<9, 3>(

               cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())

               ? emitIntrinsicWithChainErrorMessage(

                     Op, "argument out of range or not a multiple of 8", DAG)

               : SDValue();

  }

}


// Helper function that emits error message for intrinsics with void return

// value and return the chain.

static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg,

                                         SelectionDAG &DAG) {


  DAG.getContext()->emitError(Op->getOperationName(0) + ": " + ErrorMsg + ".");

  return Op.getOperand(0);

}


SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op,

                                                     SelectionDAG &DAG) const {

  SDLoc DL(Op);

  MVT GRLenVT = Subtarget.getGRLenVT();

  SDValue Chain = Op.getOperand(0);

  uint64_t IntrinsicEnum = Op.getConstantOperandVal(1);

  SDValue Op2 = Op.getOperand(2);

  const StringRef ErrorMsgOOR = "argument out of range";

  const StringRef ErrorMsgReqLA64 = "requires loongarch64";

  const StringRef ErrorMsgReqLA32 = "requires loongarch32";

  const StringRef ErrorMsgReqF = "requires basic 'f' target feature";


  switch (IntrinsicEnum) {

  default:

    // TODO: Add more Intrinsics.

    return SDValue();

  case Intrinsic::loongarch_cacop_d:

  case Intrinsic::loongarch_cacop_w: {

    if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())

      return emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG);

    if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())

      return emitIntrinsicErrorMessage(Op, ErrorMsgReqLA32, DAG);

    // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12)

    unsigned Imm1 = Op2->getAsZExtVal();

    int Imm2 = cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue();

    if (!isUInt<5>(Imm1) || !isInt<12>(Imm2))

      return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG);

    return Op;

  }

  case Intrinsic::loongarch_dbar: {

    unsigned Imm = Op2->getAsZExtVal();

    return !isUInt<15>(Imm)

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::DBAR, DL, MVT::Other, Chain,

                             DAG.getConstant(Imm, DL, GRLenVT));

  }

  case Intrinsic::loongarch_ibar: {

    unsigned Imm = Op2->getAsZExtVal();

    return !isUInt<15>(Imm)

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::IBAR, DL, MVT::Other, Chain,

                             DAG.getConstant(Imm, DL, GRLenVT));

  }

  case Intrinsic::loongarch_break: {

    unsigned Imm = Op2->getAsZExtVal();

    return !isUInt<15>(Imm)

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::BREAK, DL, MVT::Other, Chain,

                             DAG.getConstant(Imm, DL, GRLenVT));

  }

  case Intrinsic::loongarch_movgr2fcsr: {

    if (!Subtarget.hasBasicF())

      return emitIntrinsicErrorMessage(Op, ErrorMsgReqF, DAG);

    unsigned Imm = Op2->getAsZExtVal();

    return !isUInt<2>(Imm)

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::MOVGR2FCSR, DL, MVT::Other, Chain,

                             DAG.getConstant(Imm, DL, GRLenVT),

                             DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT,

                                         Op.getOperand(3)));

  }

  case Intrinsic::loongarch_syscall: {

    unsigned Imm = Op2->getAsZExtVal();

    return !isUInt<15>(Imm)

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : DAG.getNode(LoongArchISD::SYSCALL, DL, MVT::Other, Chain,

                             DAG.getConstant(Imm, DL, GRLenVT));

  }

#define IOCSRWR_CASE(NAME, NODE)                                               \

  case Intrinsic::loongarch_##NAME: {                                          \

    SDValue Op3 = Op.getOperand(3);                                            \

    return Subtarget.is64Bit()                                                 \

               ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain,        \

                             DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),  \

                             DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3))  \

               : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2,   \

                             Op3);                                             \

  }

    IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B);

    IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H);

    IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W);

#undef IOCSRWR_CASE

  case Intrinsic::loongarch_iocsrwr_d: {

    return !Subtarget.is64Bit()

               ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG)

               : DAG.getNode(LoongArchISD::IOCSRWR_D, DL, MVT::Other, Chain,

                             Op2,

                             DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,

                                         Op.getOperand(3)));

  }

#define ASRT_LE_GT_CASE(NAME)                                                  \

  case Intrinsic::loongarch_##NAME: {                                          \

    return !Subtarget.is64Bit()                                                \

               ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG)           \

               : Op;                                                           \

  }

    ASRT_LE_GT_CASE(asrtle_d)

    ASRT_LE_GT_CASE(asrtgt_d)

#undef ASRT_LE_GT_CASE

  case Intrinsic::loongarch_ldpte_d: {

    unsigned Imm = Op.getConstantOperandVal(3);

    return !Subtarget.is64Bit()

               ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG)

           : !isUInt<8>(Imm) ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

                             : Op;

  }

  case Intrinsic::loongarch_lsx_vst:

  case Intrinsic::loongarch_lasx_xvst:

    return !isInt<12>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue())

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : SDValue();

  case Intrinsic::loongarch_lasx_xvstelm_b:

    return (!isInt<8>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<5>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vstelm_b:

    return (!isInt<8>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<4>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)

               : SDValue();

  case Intrinsic::loongarch_lasx_xvstelm_h:

    return (!isShiftedInt<8, 1>(

                cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<4>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(

                     Op, "argument out of range or not a multiple of 2", DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vstelm_h:

    return (!isShiftedInt<8, 1>(

                cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<3>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(

                     Op, "argument out of range or not a multiple of 2", DAG)

               : SDValue();

  case Intrinsic::loongarch_lasx_xvstelm_w:

    return (!isShiftedInt<8, 2>(

                cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<3>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(

                     Op, "argument out of range or not a multiple of 4", DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vstelm_w:

    return (!isShiftedInt<8, 2>(

                cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<2>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(

                     Op, "argument out of range or not a multiple of 4", DAG)

               : SDValue();

  case Intrinsic::loongarch_lasx_xvstelm_d:

    return (!isShiftedInt<8, 3>(

                cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<2>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(

                     Op, "argument out of range or not a multiple of 8", DAG)

               : SDValue();

  case Intrinsic::loongarch_lsx_vstelm_d:

    return (!isShiftedInt<8, 3>(

                cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||

            !isUInt<1>(Op.getConstantOperandVal(5)))

               ? emitIntrinsicErrorMessage(

                     Op, "argument out of range or not a multiple of 8", DAG)

               : SDValue();

  }

}


SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op,

                                                     SelectionDAG &DAG) const {

  SDLoc DL(Op);

  SDValue Lo = Op.getOperand(0);

  SDValue Hi = Op.getOperand(1);

  SDValue Shamt = Op.getOperand(2);

  EVT VT = Lo.getValueType();


  // if Shamt-GRLen < 0: // Shamt < GRLen

  //   Lo = Lo << Shamt

  //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt))

  // else:

  //   Lo = 0

  //   Hi = Lo << (Shamt-GRLen)


  SDValue Zero = DAG.getConstant(0, DL, VT);

  SDValue One = DAG.getConstant(1, DL, VT);

  SDValue MinusGRLen =

      DAG.getSignedConstant(-(int)Subtarget.getGRLen(), DL, VT);

  SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT);

  SDValue ShamtMinusGRLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusGRLen);

  SDValue GRLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, GRLenMinus1);


  SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);

  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);

  SDValue ShiftRightLo =

      DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, GRLenMinus1Shamt);

  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);

  SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);

  SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusGRLen);


  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusGRLen, Zero, ISD::SETLT);


  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);

  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);


  SDValue Parts[2] = {Lo, Hi};

  return DAG.getMergeValues(Parts, DL);

}


SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op,

                                                      SelectionDAG &DAG,

                                                      bool IsSRA) const {

  SDLoc DL(Op);

  SDValue Lo = Op.getOperand(0);

  SDValue Hi = Op.getOperand(1);

  SDValue Shamt = Op.getOperand(2);

  EVT VT = Lo.getValueType();


  // SRA expansion:

  //   if Shamt-GRLen < 0: // Shamt < GRLen

  //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))

  //     Hi = Hi >>s Shamt

  //   else:

  //     Lo = Hi >>s (Shamt-GRLen);

  //     Hi = Hi >>s (GRLen-1)

  //

  // SRL expansion:

  //   if Shamt-GRLen < 0: // Shamt < GRLen

  //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))

  //     Hi = Hi >>u Shamt

  //   else:

  //     Lo = Hi >>u (Shamt-GRLen);

  //     Hi = 0;


  unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;


  SDValue Zero = DAG.getConstant(0, DL, VT);

  SDValue One = DAG.getConstant(1, DL, VT);

  SDValue MinusGRLen =

      DAG.getSignedConstant(-(int)Subtarget.getGRLen(), DL, VT);

  SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT);

  SDValue ShamtMinusGRLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusGRLen);

  SDValue GRLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, GRLenMinus1);


  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);

  SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);

  SDValue ShiftLeftHi =

      DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, GRLenMinus1Shamt);

  SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);

  SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);

  SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusGRLen);

  SDValue HiFalse =

      IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, GRLenMinus1) : Zero;


  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusGRLen, Zero, ISD::SETLT);


  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);

  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);


  SDValue Parts[2] = {Lo, Hi};

  return DAG.getMergeValues(Parts, DL);

}


// Returns the opcode of the target-specific SDNode that implements the 32-bit

// form of the given Opcode.

static LoongArchISD::NodeType getLoongArchWOpcode(unsigned Opcode) {

  switch (Opcode) {

  default:

    llvm_unreachable("Unexpected opcode");

  case ISD::SDIV:

    return LoongArchISD::DIV_W;

  case ISD::UDIV:

    return LoongArchISD::DIV_WU;

  case ISD::SREM:

    return LoongArchISD::MOD_W;

  case ISD::UREM:

    return LoongArchISD::MOD_WU;

  case ISD::SHL:

    return LoongArchISD::SLL_W;

  case ISD::SRA:

    return LoongArchISD::SRA_W;

  case ISD::SRL:

    return LoongArchISD::SRL_W;

  case ISD::ROTL:

  case ISD::ROTR:

    return LoongArchISD::ROTR_W;

  case ISD::CTTZ:

    return LoongArchISD::CTZ_W;

  case ISD::CTLZ:

    return LoongArchISD::CLZ_W;

  }

}


// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG

// node. Because i8/i16/i32 isn't a legal type for LA64, these operations would

// otherwise be promoted to i64, making it difficult to select the

// SLL_W/.../*W later one because the fact the operation was originally of

// type i8/i16/i32 is lost.

static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp,

                                   unsigned ExtOpc = ISD::ANY_EXTEND) {

  SDLoc DL(N);

  LoongArchISD::NodeType WOpcode = getLoongArchWOpcode(N->getOpcode());

  SDValue NewOp0, NewRes;


  switch (NumOp) {

  default:

    llvm_unreachable("Unexpected NumOp");

  case 1: {

    NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));

    NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0);

    break;

  }

  case 2: {

    NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));

    SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));

    if (N->getOpcode() == ISD::ROTL) {

      SDValue TmpOp = DAG.getConstant(32, DL, MVT::i64);

      NewOp1 = DAG.getNode(ISD::SUB, DL, MVT::i64, TmpOp, NewOp1);

    }

    NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);

    break;

  }

    // TODO:Handle more NumOp.

  }


  // ReplaceNodeResults requires we maintain the same type for the return

  // value.

  return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);

}


// Converts the given 32-bit operation to a i64 operation with signed extension

// semantic to reduce the signed extension instructions.

static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {

  SDLoc DL(N);

  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));

  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));

  SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);

  SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,

                               DAG.getValueType(MVT::i32));

  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);

}


// Helper function that emits error message for intrinsics with/without chain

// and return a UNDEF or and the chain as the results.

static void emitErrorAndReplaceIntrinsicResults(

    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG,

    StringRef ErrorMsg, bool WithChain = true) {

  DAG.getContext()->emitError(N->getOperationName(0) + ": " + ErrorMsg + ".");

  Results.push_back(DAG.getUNDEF(N->getValueType(0)));

  if (!WithChain)

    return;

  Results.push_back(N->getOperand(0));

}


template <unsigned N>

static void

replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl<SDValue> &Results,

                         SelectionDAG &DAG, const LoongArchSubtarget &Subtarget,

                         unsigned ResOp) {

  const StringRef ErrorMsgOOR = "argument out of range";

  unsigned Imm = Node->getConstantOperandVal(2);

  if (!isUInt<N>(Imm)) {

    emitErrorAndReplaceIntrinsicResults(Node, Results, DAG, ErrorMsgOOR,

                                        /*WithChain=*/false);

    return;

  }

  SDLoc DL(Node);

  SDValue Vec = Node->getOperand(1);


  SDValue PickElt =

      DAG.getNode(ResOp, DL, Subtarget.getGRLenVT(), Vec,

                  DAG.getConstant(Imm, DL, Subtarget.getGRLenVT()),

                  DAG.getValueType(Vec.getValueType().getVectorElementType()));

  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, Node->getValueType(0),

                                PickElt.getValue(0)));

}


static void replaceVecCondBranchResults(SDNode *N,

                                        SmallVectorImpl<SDValue> &Results,

                                        SelectionDAG &DAG,

                                        const LoongArchSubtarget &Subtarget,

                                        unsigned ResOp) {

  SDLoc DL(N);

  SDValue Vec = N->getOperand(1);


  SDValue CB = DAG.getNode(ResOp, DL, Subtarget.getGRLenVT(), Vec);

  Results.push_back(

      DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), CB.getValue(0)));

}


static void

replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl<SDValue> &Results,

                                 SelectionDAG &DAG,

                                 const LoongArchSubtarget &Subtarget) {

  switch (N->getConstantOperandVal(0)) {

  default:

    llvm_unreachable("Unexpected Intrinsic.");

  case Intrinsic::loongarch_lsx_vpickve2gr_b:

    replaceVPICKVE2GRResults<4>(N, Results, DAG, Subtarget,

                                LoongArchISD::VPICK_SEXT_ELT);

    break;

  case Intrinsic::loongarch_lsx_vpickve2gr_h:

  case Intrinsic::loongarch_lasx_xvpickve2gr_w:

    replaceVPICKVE2GRResults<3>(N, Results, DAG, Subtarget,

                                LoongArchISD::VPICK_SEXT_ELT);

    break;

  case Intrinsic::loongarch_lsx_vpickve2gr_w:

    replaceVPICKVE2GRResults<2>(N, Results, DAG, Subtarget,

                                LoongArchISD::VPICK_SEXT_ELT);

    break;

  case Intrinsic::loongarch_lsx_vpickve2gr_bu:

    replaceVPICKVE2GRResults<4>(N, Results, DAG, Subtarget,

                                LoongArchISD::VPICK_ZEXT_ELT);

    break;

  case Intrinsic::loongarch_lsx_vpickve2gr_hu:

  case Intrinsic::loongarch_lasx_xvpickve2gr_wu:

    replaceVPICKVE2GRResults<3>(N, Results, DAG, Subtarget,

                                LoongArchISD::VPICK_ZEXT_ELT);

    break;

  case Intrinsic::loongarch_lsx_vpickve2gr_wu:

    replaceVPICKVE2GRResults<2>(N, Results, DAG, Subtarget,

                                LoongArchISD::VPICK_ZEXT_ELT);

    break;

  case Intrinsic::loongarch_lsx_bz_b:

  case Intrinsic::loongarch_lsx_bz_h:

  case Intrinsic::loongarch_lsx_bz_w:

  case Intrinsic::loongarch_lsx_bz_d:

  case Intrinsic::loongarch_lasx_xbz_b:

  case Intrinsic::loongarch_lasx_xbz_h:

  case Intrinsic::loongarch_lasx_xbz_w:

  case Intrinsic::loongarch_lasx_xbz_d:

    replaceVecCondBranchResults(N, Results, DAG, Subtarget,

                                LoongArchISD::VALL_ZERO);

    break;

  case Intrinsic::loongarch_lsx_bz_v:

  case Intrinsic::loongarch_lasx_xbz_v:

    replaceVecCondBranchResults(N, Results, DAG, Subtarget,

                                LoongArchISD::VANY_ZERO);

    break;

  case Intrinsic::loongarch_lsx_bnz_b:

  case Intrinsic::loongarch_lsx_bnz_h:

  case Intrinsic::loongarch_lsx_bnz_w:

  case Intrinsic::loongarch_lsx_bnz_d:

  case Intrinsic::loongarch_lasx_xbnz_b:

  case Intrinsic::loongarch_lasx_xbnz_h:

  case Intrinsic::loongarch_lasx_xbnz_w:

  case Intrinsic::loongarch_lasx_xbnz_d:

    replaceVecCondBranchResults(N, Results, DAG, Subtarget,

                                LoongArchISD::VALL_NONZERO);

    break;

  case Intrinsic::loongarch_lsx_bnz_v:

  case Intrinsic::loongarch_lasx_xbnz_v:

    replaceVecCondBranchResults(N, Results, DAG, Subtarget,

                                LoongArchISD::VANY_NONZERO);

    break;

  }

}


void LoongArchTargetLowering::ReplaceNodeResults(

    SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {

  SDLoc DL(N);

  EVT VT = N->getValueType(0);

  switch (N->getOpcode()) {

  default:

    llvm_unreachable("Don't know how to legalize this operation");

  case ISD::ADD:

  case ISD::SUB:

    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    Results.push_back(customLegalizeToWOpWithSExt(N, DAG));

    break;

  case ISD::SDIV:

  case ISD::UDIV:

  case ISD::SREM:

  case ISD::UREM:

    assert(VT == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    Results.push_back(customLegalizeToWOp(N, DAG, 2,

                                          Subtarget.hasDiv32() && VT == MVT::i32

                                              ? ISD::ANY_EXTEND

                                              : ISD::SIGN_EXTEND));

    break;

  case ISD::SHL:

  case ISD::SRA:

  case ISD::SRL:

    assert(VT == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    if (N->getOperand(1).getOpcode() != ISD::Constant) {

      Results.push_back(customLegalizeToWOp(N, DAG, 2));

      break;

    }

    break;

  case ISD::ROTL:

  case ISD::ROTR:

    assert(VT == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    Results.push_back(customLegalizeToWOp(N, DAG, 2));

    break;

  case ISD::FP_TO_SINT: {

    assert(VT == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    SDValue Src = N->getOperand(0);

    EVT FVT = EVT::getFloatingPointVT(N->getValueSizeInBits(0));

    if (getTypeAction(*DAG.getContext(), Src.getValueType()) !=

        TargetLowering::TypeSoftenFloat) {

      if (Src.getValueType() == MVT::f16)

        Src = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);

      SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, FVT, Src);

      Results.push_back(DAG.getNode(ISD::BITCAST, DL, VT, Dst));

      return;

    }

    // If the FP type needs to be softened, emit a library call using the 'si'

    // version. If we left it to default legalization we'd end up with 'di'.

    RTLIB::Libcall LC;

    LC = RTLIB::getFPTOSINT(Src.getValueType(), VT);

    MakeLibCallOptions CallOptions;

    EVT OpVT = Src.getValueType();

    CallOptions.setTypeListBeforeSoften(OpVT, VT, true);

    SDValue Chain = SDValue();

    SDValue Result;

    std::tie(Result, Chain) =

        makeLibCall(DAG, LC, VT, Src, CallOptions, DL, Chain);

    Results.push_back(Result);

    break;

  }

  case ISD::BITCAST: {

    SDValue Src = N->getOperand(0);

    EVT SrcVT = Src.getValueType();

    if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&

        Subtarget.hasBasicF()) {

      SDValue Dst =

          DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Src);

      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Dst));

    }

    break;

  }

  case ISD::FP_TO_UINT: {

    assert(VT == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    auto &TLI = DAG.getTargetLoweringInfo();

    SDValue Tmp1, Tmp2;

    TLI.expandFP_TO_UINT(N, Tmp1, Tmp2, DAG);

    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Tmp1));

    break;

  }

  case ISD::BSWAP: {

    SDValue Src = N->getOperand(0);

    assert((VT == MVT::i16 || VT == MVT::i32) &&

           "Unexpected custom legalization");

    MVT GRLenVT = Subtarget.getGRLenVT();

    SDValue NewSrc = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Src);

    SDValue Tmp;

    switch (VT.getSizeInBits()) {

    default:

      llvm_unreachable("Unexpected operand width");

    case 16:

      Tmp = DAG.getNode(LoongArchISD::REVB_2H, DL, GRLenVT, NewSrc);

      break;

    case 32:

      // Only LA64 will get to here due to the size mismatch between VT and

      // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.

      Tmp = DAG.getNode(LoongArchISD::REVB_2W, DL, GRLenVT, NewSrc);

      break;

    }

    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp));

    break;

  }

  case ISD::BITREVERSE: {

    SDValue Src = N->getOperand(0);

    assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&

           "Unexpected custom legalization");

    MVT GRLenVT = Subtarget.getGRLenVT();

    SDValue NewSrc = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Src);

    SDValue Tmp;

    switch (VT.getSizeInBits()) {

    default:

      llvm_unreachable("Unexpected operand width");

    case 8:

      Tmp = DAG.getNode(LoongArchISD::BITREV_4B, DL, GRLenVT, NewSrc);

      break;

    case 32:

      Tmp = DAG.getNode(LoongArchISD::BITREV_W, DL, GRLenVT, NewSrc);

      break;

    }

    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp));

    break;

  }

  case ISD::CTLZ:

  case ISD::CTTZ: {

    assert(VT == MVT::i32 && Subtarget.is64Bit() &&

           "Unexpected custom legalisation");

    Results.push_back(customLegalizeToWOp(N, DAG, 1));

    break;

  }

  case ISD::INTRINSIC_W_CHAIN: {

    SDValue Chain = N->getOperand(0);

    SDValue Op2 = N->getOperand(2);

    MVT GRLenVT = Subtarget.getGRLenVT();

    const StringRef ErrorMsgOOR = "argument out of range";

    const StringRef ErrorMsgReqLA64 = "requires loongarch64";

    const StringRef ErrorMsgReqF = "requires basic 'f' target feature";


    switch (N->getConstantOperandVal(1)) {

    default:

      llvm_unreachable("Unexpected Intrinsic.");

    case Intrinsic::loongarch_movfcsr2gr: {

      if (!Subtarget.hasBasicF()) {

        emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqF);

        return;

      }

      unsigned Imm = Op2->getAsZExtVal();

      if (!isUInt<2>(Imm)) {

        emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);

        return;

      }

      SDValue MOVFCSR2GRResults = DAG.getNode(

          LoongArchISD::MOVFCSR2GR, SDLoc(N), {MVT::i64, MVT::Other},

          {Chain, DAG.getConstant(Imm, DL, GRLenVT)});

      Results.push_back(

          DAG.getNode(ISD::TRUNCATE, DL, VT, MOVFCSR2GRResults.getValue(0)));

      Results.push_back(MOVFCSR2GRResults.getValue(1));

      break;

    }

#define CRC_CASE_EXT_BINARYOP(NAME, NODE)                                      \

  case Intrinsic::loongarch_##NAME: {                                          \

    SDValue NODE = DAG.getNode(                                                \

        LoongArchISD::NODE, DL, {MVT::i64, MVT::Other},                        \

        {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),               \

         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))});       \

    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0)));   \

    Results.push_back(NODE.getValue(1));                                       \

    break;                                                                     \

  }

      CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)

      CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W)

      CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W)

      CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W)

      CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W)

      CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W)

#undef CRC_CASE_EXT_BINARYOP


#define CRC_CASE_EXT_UNARYOP(NAME, NODE)                                       \

  case Intrinsic::loongarch_##NAME: {                                          \

    SDValue NODE = DAG.getNode(                                                \

        LoongArchISD::NODE, DL, {MVT::i64, MVT::Other},                        \

        {Chain, Op2,                                                           \

         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))});       \

    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0)));   \

    Results.push_back(NODE.getValue(1));                                       \

    break;                                                                     \

  }

      CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)

      CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W)

#undef CRC_CASE_EXT_UNARYOP

#define CSR_CASE(ID)                                                           \

  case Intrinsic::loongarch_##ID: {                                            \

    if (!Subtarget.is64Bit())                                                  \

      emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64);   \

    break;                                                                     \

  }

      CSR_CASE(csrrd_d);

      CSR_CASE(csrwr_d);

      CSR_CASE(csrxchg_d);

      CSR_CASE(iocsrrd_d);

#undef CSR_CASE

    case Intrinsic::loongarch_csrrd_w: {

      unsigned Imm = Op2->getAsZExtVal();

      if (!isUInt<14>(Imm)) {

        emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);

        return;

      }

      SDValue CSRRDResults =

          DAG.getNode(LoongArchISD::CSRRD, DL, {GRLenVT, MVT::Other},

                      {Chain, DAG.getConstant(Imm, DL, GRLenVT)});

      Results.push_back(

          DAG.getNode(ISD::TRUNCATE, DL, VT, CSRRDResults.getValue(0)));

      Results.push_back(CSRRDResults.getValue(1));

      break;

    }

    case Intrinsic::loongarch_csrwr_w: {

      unsigned Imm = N->getConstantOperandVal(3);

      if (!isUInt<14>(Imm)) {

        emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);

        return;

      }

      SDValue CSRWRResults =

          DAG.getNode(LoongArchISD::CSRWR, DL, {GRLenVT, MVT::Other},

                      {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),

                       DAG.getConstant(Imm, DL, GRLenVT)});

      Results.push_back(

          DAG.getNode(ISD::TRUNCATE, DL, VT, CSRWRResults.getValue(0)));

      Results.push_back(CSRWRResults.getValue(1));

      break;

    }

    case Intrinsic::loongarch_csrxchg_w: {

      unsigned Imm = N->getConstantOperandVal(4);

      if (!isUInt<14>(Imm)) {

        emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);

        return;

      }

      SDValue CSRXCHGResults = DAG.getNode(

          LoongArchISD::CSRXCHG, DL, {GRLenVT, MVT::Other},

          {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),

           DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3)),

           DAG.getConstant(Imm, DL, GRLenVT)});

      Results.push_back(

          DAG.getNode(ISD::TRUNCATE, DL, VT, CSRXCHGResults.getValue(0)));

      Results.push_back(CSRXCHGResults.getValue(1));

      break;

    }

#define IOCSRRD_CASE(NAME, NODE)                                               \

  case Intrinsic::loongarch_##NAME: {                                          \

    SDValue IOCSRRDResults =                                                   \

        DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other},            \

                    {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \

    Results.push_back(                                                         \

        DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0)));       \

    Results.push_back(IOCSRRDResults.getValue(1));                             \

    break;                                                                     \

  }

      IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);

      IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);

      IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);

#undef IOCSRRD_CASE

    case Intrinsic::loongarch_cpucfg: {

      SDValue CPUCFGResults =

          DAG.getNode(LoongArchISD::CPUCFG, DL, {GRLenVT, MVT::Other},

                      {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)});

      Results.push_back(

          DAG.getNode(ISD::TRUNCATE, DL, VT, CPUCFGResults.getValue(0)));

      Results.push_back(CPUCFGResults.getValue(1));

      break;

    }

    case Intrinsic::loongarch_lddir_d: {

      if (!Subtarget.is64Bit()) {

        emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64);

        return;

      }

      break;

    }

    }

    break;

  }

  case ISD::READ_REGISTER: {

    if (Subtarget.is64Bit())

      DAG.getContext()->emitError(

          "On LA64, only 64-bit registers can be read.");

    else

      DAG.getContext()->emitError(

          "On LA32, only 32-bit registers can be read.");

    Results.push_back(DAG.getUNDEF(VT));

    Results.push_back(N->getOperand(0));

    break;

  }

  case ISD::INTRINSIC_WO_CHAIN: {

    replaceINTRINSIC_WO_CHAINResults(N, Results, DAG, Subtarget);

    break;

  }

  case ISD::LROUND: {

    SDValue Op0 = N->getOperand(0);

    EVT OpVT = Op0.getValueType();

    RTLIB::Libcall LC =

        OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;

    MakeLibCallOptions CallOptions;

    CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64, true);

    SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;

    Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);

    Results.push_back(Result);

    break;

  }

  }

}


static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,

                                 TargetLowering::DAGCombinerInfo &DCI,

                                 const LoongArchSubtarget &Subtarget) {

  if (DCI.isBeforeLegalizeOps())

    return SDValue();


  SDValue FirstOperand = N->getOperand(0);

  SDValue SecondOperand = N->getOperand(1);

  unsigned FirstOperandOpc = FirstOperand.getOpcode();

  EVT ValTy = N->getValueType(0);

  SDLoc DL(N);

  uint64_t lsb, msb;

  unsigned SMIdx, SMLen;

  ConstantSDNode *CN;

  SDValue NewOperand;

  MVT GRLenVT = Subtarget.getGRLenVT();


  // Op's second operand must be a shifted mask.

  if (!(CN = dyn_cast<ConstantSDNode>(SecondOperand)) ||

      !isShiftedMask_64(CN->getZExtValue(), SMIdx, SMLen))

    return SDValue();


  if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {

    // Pattern match BSTRPICK.

    //  $dst = and ((sra or srl) $src , lsb), (2**len - 1)

    //  => BSTRPICK $dst, $src, msb, lsb

    //  where msb = lsb + len - 1


    // The second operand of the shift must be an immediate.

    if (!(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))))

      return SDValue();


    lsb = CN->getZExtValue();


    // Return if the shifted mask does not start at bit 0 or the sum of its

    // length and lsb exceeds the word's size.

    if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits())

      return SDValue();


    NewOperand = FirstOperand.getOperand(0);

  } else {

    // Pattern match BSTRPICK.

    //  $dst = and $src, (2**len- 1) , if len > 12

    //  => BSTRPICK $dst, $src, msb, lsb

    //  where lsb = 0 and msb = len - 1


    // If the mask is <= 0xfff, andi can be used instead.

    if (CN->getZExtValue() <= 0xfff)

      return SDValue();


    // Return if the MSB exceeds.

    if (SMIdx + SMLen > ValTy.getSizeInBits())

      return SDValue();


    if (SMIdx > 0) {

      // Omit if the constant has more than 2 uses. This a conservative

      // decision. Whether it is a win depends on the HW microarchitecture.

      // However it should always be better for 1 and 2 uses.

      if (CN->use_size() > 2)

        return SDValue();

      // Return if the constant can be composed by a single LU12I.W.

      if ((CN->getZExtValue() & 0xfff) == 0)

        return SDValue();

      // Return if the constand can be composed by a single ADDI with

      // the zero register.

      if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0)

        return SDValue();

    }


    lsb = SMIdx;

    NewOperand = FirstOperand;

  }


  msb = lsb + SMLen - 1;

  SDValue NR0 = DAG.getNode(LoongArchISD::BSTRPICK, DL, ValTy, NewOperand,

                            DAG.getConstant(msb, DL, GRLenVT),

                            DAG.getConstant(lsb, DL, GRLenVT));

  if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL || lsb == 0)

    return NR0;

  // Try to optimize to

  //   bstrpick $Rd, $Rs, msb, lsb

  //   slli     $Rd, $Rd, lsb

  return DAG.getNode(ISD::SHL, DL, ValTy, NR0,

                     DAG.getConstant(lsb, DL, GRLenVT));

}


static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,

                                 TargetLowering::DAGCombinerInfo &DCI,

                                 const LoongArchSubtarget &Subtarget) {

  if (DCI.isBeforeLegalizeOps())

    return SDValue();


  // $dst = srl (and $src, Mask), Shamt

  // =>

  // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt

  // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1

  //


  SDValue FirstOperand = N->getOperand(0);

  ConstantSDNode *CN;

  EVT ValTy = N->getValueType(0);

  SDLoc DL(N);

  MVT GRLenVT = Subtarget.getGRLenVT();

  unsigned MaskIdx, MaskLen;

  uint64_t Shamt;


  // The first operand must be an AND and the second operand of the AND must be

  // a shifted mask.

  if (FirstOperand.getOpcode() != ISD::AND ||

      !(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))) ||

      !isShiftedMask_64(CN->getZExtValue(), MaskIdx, MaskLen))

    return SDValue();


  // The second operand (shift amount) must be an immediate.

  if (!(CN = dyn_cast<ConstantSDNode>(N->getOperand(1))))

    return SDValue();


  Shamt = CN->getZExtValue();

  if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)

    return DAG.getNode(LoongArchISD::BSTRPICK, DL, ValTy,

                       FirstOperand->getOperand(0),

                       DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT),

                       DAG.getConstant(Shamt, DL, GRLenVT));


  return SDValue();

}


static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,

                                TargetLowering::DAGCombinerInfo &DCI,

                                const LoongArchSubtarget &Subtarget) {

  MVT GRLenVT = Subtarget.getGRLenVT();

  EVT ValTy = N->getValueType(0);

  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);

  ConstantSDNode *CN0, *CN1;

  SDLoc DL(N);

  unsigned ValBits = ValTy.getSizeInBits();

  unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;

  unsigned Shamt;

  bool SwapAndRetried = false;


  if (DCI.isBeforeLegalizeOps())

    return SDValue();


  if (ValBits != 32 && ValBits != 64)

    return SDValue();


Retry:

  // 1st pattern to match BSTRINS:

  //  R = or (and X, mask0), (and (shl Y, lsb), mask1)

  //  where mask1 = (2**size - 1) << lsb, mask0 = ~mask1

  //  =>

  //  R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)

  if (N0.getOpcode() == ISD::AND &&

      (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&

      isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&

      N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL &&

      (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&

      isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) &&

      MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&

      (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&

      (Shamt = CN1->getZExtValue()) == MaskIdx0 &&

      (MaskIdx0 + MaskLen0 <= ValBits)) {

    LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n");

    return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),

                       N1.getOperand(0).getOperand(0),

                       DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT),

                       DAG.getConstant(MaskIdx0, DL, GRLenVT));

  }


  // 2nd pattern to match BSTRINS:

  //  R = or (and X, mask0), (shl (and Y, mask1), lsb)

  //  where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb)

  //  =>

  //  R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)

  if (N0.getOpcode() == ISD::AND &&

      (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&

      isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&

      N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::AND &&

      (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&

      (Shamt = CN1->getZExtValue()) == MaskIdx0 &&

      (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&

      isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) &&

      MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&

      (MaskIdx0 + MaskLen0 <= ValBits)) {

    LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n");

    return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),

                       N1.getOperand(0).getOperand(0),

                       DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT),

                       DAG.getConstant(MaskIdx0, DL, GRLenVT));

  }


  // 3rd pattern to match BSTRINS:

  //  R = or (and X, mask0), (and Y, mask1)

  //  where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0

  //  =>

  //  R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb

  //  where msb = lsb + size - 1

  if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&

      (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&

      isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&

      (MaskIdx0 + MaskLen0 <= 64) &&

      (CN1 = dyn_cast<ConstantSDNode>(N1->getOperand(1))) &&

      (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {

    LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n");

    return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),

                       DAG.getNode(ISD::SRL, DL, N1->getValueType(0), N1,

                                   DAG.getConstant(MaskIdx0, DL, GRLenVT)),

                       DAG.getConstant(ValBits == 32

                                           ? (MaskIdx0 + (MaskLen0 & 31) - 1)

                                           : (MaskIdx0 + MaskLen0 - 1),

                                       DL, GRLenVT),

                       DAG.getConstant(MaskIdx0, DL, GRLenVT));

  }


  // 4th pattern to match BSTRINS:

  //  R = or (and X, mask), (shl Y, shamt)

  //  where mask = (2**shamt - 1)

  //  =>

  //  R = BSTRINS X, Y, ValBits - 1, shamt

  //  where ValBits = 32 or 64

  if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL &&

      (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&

      isShiftedMask_64(CN0->getZExtValue(), MaskIdx0, MaskLen0) &&

      MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&

      (Shamt = CN1->getZExtValue()) == MaskLen0 &&

      (MaskIdx0 + MaskLen0 <= ValBits)) {

    LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n");

    return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),

                       N1.getOperand(0),

                       DAG.getConstant((ValBits - 1), DL, GRLenVT),

                       DAG.getConstant(Shamt, DL, GRLenVT));

  }


  // 5th pattern to match BSTRINS:

  //  R = or (and X, mask), const

  //  where ~mask = (2**size - 1) << lsb, mask & const = 0

  //  =>

  //  R = BSTRINS X, (const >> lsb), msb, lsb

  //  where msb = lsb + size - 1

  if (N0.getOpcode() == ISD::AND &&

      (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&

      isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&

      (CN1 = dyn_cast<ConstantSDNode>(N1)) &&

      (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {

    LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n");

    return DAG.getNode(

        LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),

        DAG.getConstant(CN1->getSExtValue() >> MaskIdx0, DL, ValTy),

        DAG.getConstant(ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)

                                      : (MaskIdx0 + MaskLen0 - 1),

                        DL, GRLenVT),

        DAG.getConstant(MaskIdx0, DL, GRLenVT));

  }


  // 6th pattern.

  // a = b | ((c & mask) << shamt), where all positions in b to be overwritten

  // by the incoming bits are known to be zero.

  // =>

  // a = BSTRINS b, c, shamt + MaskLen - 1, shamt

  //

  // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th

  // pattern is more common than the 1st. So we put the 1st before the 6th in

  // order to match as many nodes as possible.

  ConstantSDNode *CNMask, *CNShamt;

  unsigned MaskIdx, MaskLen;

  if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::AND &&

      (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&

      isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) &&

      MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&

      CNShamt->getZExtValue() + MaskLen <= ValBits) {

    Shamt = CNShamt->getZExtValue();

    APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt);

    if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) {

      LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n");

      return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0,

                         N1.getOperand(0).getOperand(0),

                         DAG.getConstant(Shamt + MaskLen - 1, DL, GRLenVT),

                         DAG.getConstant(Shamt, DL, GRLenVT));

    }

  }


  // 7th pattern.

  // a = b | ((c << shamt) & shifted_mask), where all positions in b to be

  // overwritten by the incoming bits are known to be zero.

  // =>

  // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx

  //

  // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd

  // before the 7th in order to match as many nodes as possible.

  if (N1.getOpcode() == ISD::AND &&

      (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&

      isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) &&

      N1.getOperand(0).getOpcode() == ISD::SHL &&

      (CNShamt = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&

      CNShamt->getZExtValue() == MaskIdx) {

    APInt ShMask(ValBits, CNMask->getZExtValue());

    if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) {

      LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n");

      return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0,

                         N1.getOperand(0).getOperand(0),

                         DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT),

                         DAG.getConstant(MaskIdx, DL, GRLenVT));

    }

  }


  // (or a, b) and (or b, a) are equivalent, so swap the operands and retry.

  if (!SwapAndRetried) {

    std::swap(N0, N1);

    SwapAndRetried = true;

    goto Retry;

  }


  SwapAndRetried = false;

Retry2:

  // 8th pattern.

  // a = b | (c & shifted_mask), where all positions in b to be overwritten by

  // the incoming bits are known to be zero.

  // =>

  // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx

  //

  // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So

  // we put it here in order to match as many nodes as possible or generate less

  // instructions.

  if (N1.getOpcode() == ISD::AND &&

      (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&

      isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen)) {

    APInt ShMask(ValBits, CNMask->getZExtValue());

    if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) {

      LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n");

      return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0,

                         DAG.getNode(ISD::SRL, DL, N1->getValueType(0),

                                     N1->getOperand(0),

                                     DAG.getConstant(MaskIdx, DL, GRLenVT)),

                         DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT),

                         DAG.getConstant(MaskIdx, DL, GRLenVT));

    }

  }

  // Swap N0/N1 and retry.

  if (!SwapAndRetried) {

    std::swap(N0, N1);

    SwapAndRetried = true;

    goto Retry2;

  }


  return SDValue();

}


static bool checkValueWidth(SDValue V, ISD::LoadExtType &ExtType) {

  ExtType = ISD::NON_EXTLOAD;


  switch (V.getNode()->getOpcode()) {

  case ISD::LOAD: {

    LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());

    if ((LoadNode->getMemoryVT() == MVT::i8) ||

        (LoadNode->getMemoryVT() == MVT::i16)) {

      ExtType = LoadNode->getExtensionType();

      return true;

    }

    return false;

  }

  case ISD::AssertSext: {

    VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));

    if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {

      ExtType = ISD::SEXTLOAD;

      return true;

    }

    return false;

  }

  case ISD::AssertZext: {

    VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));

    if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {

      ExtType = ISD::ZEXTLOAD;

      return true;

    }

    return false;

  }

  default:

    return false;

  }


  return false;

}


// Eliminate redundant truncation and zero-extension nodes.

// * Case 1:

//  +------------+ +------------+ +------------+

//  |   Input1   | |   Input2   | |     CC     |

//  +------------+ +------------+ +------------+

//         |              |              |

//         V              V              +----+

//  +------------+ +------------+             |

//  |  TRUNCATE  | |  TRUNCATE  |             |

//  +------------+ +------------+             |

//         |              |                   |

//         V              V                   |

//  +------------+ +------------+             |

//  |  ZERO_EXT  | |  ZERO_EXT  |             |

//  +------------+ +------------+             |

//         |              |                   |

//         |              +-------------+     |

//         V              V             |     |

//        +----------------+            |     |

//        |      AND       |            |     |

//        +----------------+            |     |

//                |                     |     |

//                +---------------+     |     |

//                                |     |     |

//                                V     V     V

//                               +-------------+

//                               |     CMP     |

//                               +-------------+

// * Case 2:

//  +------------+ +------------+ +-------------+ +------------+ +------------+

//  |   Input1   | |   Input2   | | Constant -1 | | Constant 0 | |     CC     |

//  +------------+ +------------+ +-------------+ +------------+ +------------+

//         |              |             |               |               |

//         V              |             |               |               |

//  +------------+        |             |               |               |

//  |     XOR    |<---------------------+               |               |

//  +------------+        |                             |               |

//         |              |                             |               |

//         V              V             +---------------+               |

//  +------------+ +------------+       |                               |

//  |  TRUNCATE  | |  TRUNCATE  |       |     +-------------------------+

//  +------------+ +------------+       |     |

//         |              |             |     |

//         V              V             |     |

//  +------------+ +------------+       |     |

//  |  ZERO_EXT  | |  ZERO_EXT  |       |     |

//  +------------+ +------------+       |     |

//         |              |             |     |

//         V              V             |     |

//        +----------------+            |     |

//        |      AND       |            |     |

//        +----------------+            |     |

//                |                     |     |

//                +---------------+     |     |

//                                |     |     |

//                                V     V     V

//                               +-------------+

//                               |     CMP     |

//                               +-------------+

static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,

                                   TargetLowering::DAGCombinerInfo &DCI,

                                   const LoongArchSubtarget &Subtarget) {

  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();


  SDNode *AndNode = N->getOperand(0).getNode();

  if (AndNode->getOpcode() != ISD::AND)

    return SDValue();


  SDValue AndInputValue2 = AndNode->getOperand(1);

  if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)

    return SDValue();


  SDValue CmpInputValue = N->getOperand(1);

  SDValue AndInputValue1 = AndNode->getOperand(0);

  if (AndInputValue1.getOpcode() == ISD::XOR) {

    if (CC != ISD::SETEQ && CC != ISD::SETNE)

      return SDValue();

    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndInputValue1.getOperand(1));

    if (!CN || CN->getSExtValue() != -1)

      return SDValue();

    CN = dyn_cast<ConstantSDNode>(CmpInputValue);

    if (!CN || CN->getSExtValue() != 0)

      return SDValue();

    AndInputValue1 = AndInputValue1.getOperand(0);

    if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)

      return SDValue();

  } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {

    if (AndInputValue2 != CmpInputValue)

      return SDValue();

  } else {

    return SDValue();

  }


  SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(0);

  if (TruncValue1.getOpcode() != ISD::TRUNCATE)

    return SDValue();


  SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(0);

  if (TruncValue2.getOpcode() != ISD::TRUNCATE)

    return SDValue();


  SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(0);

  SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(0);

  ISD::LoadExtType ExtType1;

  ISD::LoadExtType ExtType2;


  if (!checkValueWidth(TruncInputValue1, ExtType1) ||

      !checkValueWidth(TruncInputValue2, ExtType2))

    return SDValue();


  if (TruncInputValue1->getValueType(0) != TruncInputValue2->getValueType(0) ||

      AndNode->getValueType(0) != TruncInputValue1->getValueType(0))

    return SDValue();


  if ((ExtType2 != ISD::ZEXTLOAD) &&

      ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))

    return SDValue();


  // These truncation and zero-extension nodes are not necessary, remove them.

  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N), AndNode->getValueType(0),

                               TruncInputValue1, TruncInputValue2);

  SDValue NewSetCC =

      DAG.getSetCC(SDLoc(N), N->getValueType(0), NewAnd, TruncInputValue2, CC);

  DAG.ReplaceAllUsesWith(N, NewSetCC.getNode());

  return SDValue(N, 0);

}


// Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.

static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG,

                                      TargetLowering::DAGCombinerInfo &DCI,

                                      const LoongArchSubtarget &Subtarget) {

  if (DCI.isBeforeLegalizeOps())

    return SDValue();


  SDValue Src = N->getOperand(0);

  if (Src.getOpcode() != LoongArchISD::REVB_2W)

    return SDValue();


  return DAG.getNode(LoongArchISD::BITREV_4B, SDLoc(N), N->getValueType(0),

                     Src.getOperand(0));

}


template <unsigned N>

static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp,

                                       SelectionDAG &DAG,

                                       const LoongArchSubtarget &Subtarget,

                                       bool IsSigned = false) {

  SDLoc DL(Node);

  auto *CImm = cast<ConstantSDNode>(Node->getOperand(ImmOp));

  // Check the ImmArg.

  if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||

      (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {

    DAG.getContext()->emitError(Node->getOperationName(0) +

                                ": argument out of range.");

    return DAG.getNode(ISD::UNDEF, DL, Subtarget.getGRLenVT());

  }

  return DAG.getConstant(CImm->getZExtValue(), DL, Subtarget.getGRLenVT());

}


template <unsigned N>

static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp,

                                   SelectionDAG &DAG, bool IsSigned = false) {

  SDLoc DL(Node);

  EVT ResTy = Node->getValueType(0);

  auto *CImm = cast<ConstantSDNode>(Node->getOperand(ImmOp));


  // Check the ImmArg.

  if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||

      (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {

    DAG.getContext()->emitError(Node->getOperationName(0) +

                                ": argument out of range.");

    return DAG.getNode(ISD::UNDEF, DL, ResTy);

  }

  return DAG.getConstant(

      APInt(ResTy.getScalarType().getSizeInBits(),

            IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),

      DL, ResTy);

}


static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG) {

  SDLoc DL(Node);

  EVT ResTy = Node->getValueType(0);

  SDValue Vec = Node->getOperand(2);

  SDValue Mask = DAG.getConstant(Vec.getScalarValueSizeInBits() - 1, DL, ResTy);

  return DAG.getNode(ISD::AND, DL, ResTy, Vec, Mask);

}


static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG) {

  SDLoc DL(Node);

  EVT ResTy = Node->getValueType(0);

  SDValue One = DAG.getConstant(1, DL, ResTy);

  SDValue Bit =

      DAG.getNode(ISD::SHL, DL, ResTy, One, truncateVecElts(Node, DAG));


  return DAG.getNode(ISD::AND, DL, ResTy, Node->getOperand(1),

                     DAG.getNOT(DL, Bit, ResTy));

}


template <unsigned N>

static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG) {

  SDLoc DL(Node);

  EVT ResTy = Node->getValueType(0);

  auto *CImm = cast<ConstantSDNode>(Node->getOperand(2));

  // Check the unsigned ImmArg.

  if (!isUInt<N>(CImm->getZExtValue())) {

    DAG.getContext()->emitError(Node->getOperationName(0) +

                                ": argument out of range.");

    return DAG.getNode(ISD::UNDEF, DL, ResTy);

  }


  APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();

  SDValue Mask = DAG.getConstant(~BitImm, DL, ResTy);


  return DAG.getNode(ISD::AND, DL, ResTy, Node->getOperand(1), Mask);

}


template <unsigned N>

static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG) {

  SDLoc DL(Node);

  EVT ResTy = Node->getValueType(0);

  auto *CImm = cast<ConstantSDNode>(Node->getOperand(2));

  // Check the unsigned ImmArg.

  if (!isUInt<N>(CImm->getZExtValue())) {

    DAG.getContext()->emitError(Node->getOperationName(0) +

                                ": argument out of range.");

    return DAG.getNode(ISD::UNDEF, DL, ResTy);

  }


  APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();

  SDValue BitImm = DAG.getConstant(Imm, DL, ResTy);

  return DAG.getNode(ISD::OR, DL, ResTy, Node->getOperand(1), BitImm);

}


template <unsigned N>

static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG) {

  SDLoc DL(Node);

  EVT ResTy = Node->getValueType(0);

  auto *CImm = cast<ConstantSDNode>(Node->getOperand(2));

  // Check the unsigned ImmArg.

  if (!isUInt<N>(CImm->getZExtValue())) {

    DAG.getContext()->emitError(Node->getOperationName(0) +

                                ": argument out of range.");

    return DAG.getNode(ISD::UNDEF, DL, ResTy);

  }


  APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();

  SDValue BitImm = DAG.getConstant(Imm, DL, ResTy);

  return DAG.getNode(ISD::XOR, DL, ResTy, Node->getOperand(1), BitImm);

}


static SDValue

performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,

                                 TargetLowering::DAGCombinerInfo &DCI,

                                 const LoongArchSubtarget &Subtarget) {

  SDLoc DL(N);

  switch (N->getConstantOperandVal(0)) {

  default:

    break;

  case Intrinsic::loongarch_lsx_vadd_b:

  case Intrinsic::loongarch_lsx_vadd_h:

  case Intrinsic::loongarch_lsx_vadd_w:

  case Intrinsic::loongarch_lsx_vadd_d:

  case Intrinsic::loongarch_lasx_xvadd_b:

  case Intrinsic::loongarch_lasx_xvadd_h:

  case Intrinsic::loongarch_lasx_xvadd_w:

  case Intrinsic::loongarch_lasx_xvadd_d:

    return DAG.getNode(ISD::ADD, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vaddi_bu:

  case Intrinsic::loongarch_lsx_vaddi_hu:

  case Intrinsic::loongarch_lsx_vaddi_wu:

  case Intrinsic::loongarch_lsx_vaddi_du:

  case Intrinsic::loongarch_lasx_xvaddi_bu:

  case Intrinsic::loongarch_lasx_xvaddi_hu:

  case Intrinsic::loongarch_lasx_xvaddi_wu:

  case Intrinsic::loongarch_lasx_xvaddi_du:

    return DAG.getNode(ISD::ADD, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsub_b:

  case Intrinsic::loongarch_lsx_vsub_h:

  case Intrinsic::loongarch_lsx_vsub_w:

  case Intrinsic::loongarch_lsx_vsub_d:

  case Intrinsic::loongarch_lasx_xvsub_b:

  case Intrinsic::loongarch_lasx_xvsub_h:

  case Intrinsic::loongarch_lasx_xvsub_w:

  case Intrinsic::loongarch_lasx_xvsub_d:

    return DAG.getNode(ISD::SUB, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vsubi_bu:

  case Intrinsic::loongarch_lsx_vsubi_hu:

  case Intrinsic::loongarch_lsx_vsubi_wu:

  case Intrinsic::loongarch_lsx_vsubi_du:

  case Intrinsic::loongarch_lasx_xvsubi_bu:

  case Intrinsic::loongarch_lasx_xvsubi_hu:

  case Intrinsic::loongarch_lasx_xvsubi_wu:

  case Intrinsic::loongarch_lasx_xvsubi_du:

    return DAG.getNode(ISD::SUB, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vneg_b:

  case Intrinsic::loongarch_lsx_vneg_h:

  case Intrinsic::loongarch_lsx_vneg_w:

  case Intrinsic::loongarch_lsx_vneg_d:

  case Intrinsic::loongarch_lasx_xvneg_b:

  case Intrinsic::loongarch_lasx_xvneg_h:

  case Intrinsic::loongarch_lasx_xvneg_w:

  case Intrinsic::loongarch_lasx_xvneg_d:

    return DAG.getNode(

        ISD::SUB, DL, N->getValueType(0),

        DAG.getConstant(

            APInt(N->getValueType(0).getScalarType().getSizeInBits(), 0,

                  /*isSigned=*/true),

            SDLoc(N), N->getValueType(0)),

        N->getOperand(1));

  case Intrinsic::loongarch_lsx_vmax_b:

  case Intrinsic::loongarch_lsx_vmax_h:

  case Intrinsic::loongarch_lsx_vmax_w:

  case Intrinsic::loongarch_lsx_vmax_d:

  case Intrinsic::loongarch_lasx_xvmax_b:

  case Intrinsic::loongarch_lasx_xvmax_h:

  case Intrinsic::loongarch_lasx_xvmax_w:

  case Intrinsic::loongarch_lasx_xvmax_d:

    return DAG.getNode(ISD::SMAX, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmax_bu:

  case Intrinsic::loongarch_lsx_vmax_hu:

  case Intrinsic::loongarch_lsx_vmax_wu:

  case Intrinsic::loongarch_lsx_vmax_du:

  case Intrinsic::loongarch_lasx_xvmax_bu:

  case Intrinsic::loongarch_lasx_xvmax_hu:

  case Intrinsic::loongarch_lasx_xvmax_wu:

  case Intrinsic::loongarch_lasx_xvmax_du:

    return DAG.getNode(ISD::UMAX, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmaxi_b:

  case Intrinsic::loongarch_lsx_vmaxi_h:

  case Intrinsic::loongarch_lsx_vmaxi_w:

  case Intrinsic::loongarch_lsx_vmaxi_d:

  case Intrinsic::loongarch_lasx_xvmaxi_b:

  case Intrinsic::loongarch_lasx_xvmaxi_h:

  case Intrinsic::loongarch_lasx_xvmaxi_w:

  case Intrinsic::loongarch_lasx_xvmaxi_d:

    return DAG.getNode(ISD::SMAX, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG, /*IsSigned=*/true));

  case Intrinsic::loongarch_lsx_vmaxi_bu:

  case Intrinsic::loongarch_lsx_vmaxi_hu:

  case Intrinsic::loongarch_lsx_vmaxi_wu:

  case Intrinsic::loongarch_lsx_vmaxi_du:

  case Intrinsic::loongarch_lasx_xvmaxi_bu:

  case Intrinsic::loongarch_lasx_xvmaxi_hu:

  case Intrinsic::loongarch_lasx_xvmaxi_wu:

  case Intrinsic::loongarch_lasx_xvmaxi_du:

    return DAG.getNode(ISD::UMAX, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vmin_b:

  case Intrinsic::loongarch_lsx_vmin_h:

  case Intrinsic::loongarch_lsx_vmin_w:

  case Intrinsic::loongarch_lsx_vmin_d:

  case Intrinsic::loongarch_lasx_xvmin_b:

  case Intrinsic::loongarch_lasx_xvmin_h:

  case Intrinsic::loongarch_lasx_xvmin_w:

  case Intrinsic::loongarch_lasx_xvmin_d:

    return DAG.getNode(ISD::SMIN, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmin_bu:

  case Intrinsic::loongarch_lsx_vmin_hu:

  case Intrinsic::loongarch_lsx_vmin_wu:

  case Intrinsic::loongarch_lsx_vmin_du:

  case Intrinsic::loongarch_lasx_xvmin_bu:

  case Intrinsic::loongarch_lasx_xvmin_hu:

  case Intrinsic::loongarch_lasx_xvmin_wu:

  case Intrinsic::loongarch_lasx_xvmin_du:

    return DAG.getNode(ISD::UMIN, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmini_b:

  case Intrinsic::loongarch_lsx_vmini_h:

  case Intrinsic::loongarch_lsx_vmini_w:

  case Intrinsic::loongarch_lsx_vmini_d:

  case Intrinsic::loongarch_lasx_xvmini_b:

  case Intrinsic::loongarch_lasx_xvmini_h:

  case Intrinsic::loongarch_lasx_xvmini_w:

  case Intrinsic::loongarch_lasx_xvmini_d:

    return DAG.getNode(ISD::SMIN, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG, /*IsSigned=*/true));

  case Intrinsic::loongarch_lsx_vmini_bu:

  case Intrinsic::loongarch_lsx_vmini_hu:

  case Intrinsic::loongarch_lsx_vmini_wu:

  case Intrinsic::loongarch_lsx_vmini_du:

  case Intrinsic::loongarch_lasx_xvmini_bu:

  case Intrinsic::loongarch_lasx_xvmini_hu:

  case Intrinsic::loongarch_lasx_xvmini_wu:

  case Intrinsic::loongarch_lasx_xvmini_du:

    return DAG.getNode(ISD::UMIN, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vmul_b:

  case Intrinsic::loongarch_lsx_vmul_h:

  case Intrinsic::loongarch_lsx_vmul_w:

  case Intrinsic::loongarch_lsx_vmul_d:

  case Intrinsic::loongarch_lasx_xvmul_b:

  case Intrinsic::loongarch_lasx_xvmul_h:

  case Intrinsic::loongarch_lasx_xvmul_w:

  case Intrinsic::loongarch_lasx_xvmul_d:

    return DAG.getNode(ISD::MUL, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmadd_b:

  case Intrinsic::loongarch_lsx_vmadd_h:

  case Intrinsic::loongarch_lsx_vmadd_w:

  case Intrinsic::loongarch_lsx_vmadd_d:

  case Intrinsic::loongarch_lasx_xvmadd_b:

  case Intrinsic::loongarch_lasx_xvmadd_h:

  case Intrinsic::loongarch_lasx_xvmadd_w:

  case Intrinsic::loongarch_lasx_xvmadd_d: {

    EVT ResTy = N->getValueType(0);

    return DAG.getNode(ISD::ADD, SDLoc(N), ResTy, N->getOperand(1),

                       DAG.getNode(ISD::MUL, SDLoc(N), ResTy, N->getOperand(2),

                                   N->getOperand(3)));

  }

  case Intrinsic::loongarch_lsx_vmsub_b:

  case Intrinsic::loongarch_lsx_vmsub_h:

  case Intrinsic::loongarch_lsx_vmsub_w:

  case Intrinsic::loongarch_lsx_vmsub_d:

  case Intrinsic::loongarch_lasx_xvmsub_b:

  case Intrinsic::loongarch_lasx_xvmsub_h:

  case Intrinsic::loongarch_lasx_xvmsub_w:

  case Intrinsic::loongarch_lasx_xvmsub_d: {

    EVT ResTy = N->getValueType(0);

    return DAG.getNode(ISD::SUB, SDLoc(N), ResTy, N->getOperand(1),

                       DAG.getNode(ISD::MUL, SDLoc(N), ResTy, N->getOperand(2),

                                   N->getOperand(3)));

  }

  case Intrinsic::loongarch_lsx_vdiv_b:

  case Intrinsic::loongarch_lsx_vdiv_h:

  case Intrinsic::loongarch_lsx_vdiv_w:

  case Intrinsic::loongarch_lsx_vdiv_d:

  case Intrinsic::loongarch_lasx_xvdiv_b:

  case Intrinsic::loongarch_lasx_xvdiv_h:

  case Intrinsic::loongarch_lasx_xvdiv_w:

  case Intrinsic::loongarch_lasx_xvdiv_d:

    return DAG.getNode(ISD::SDIV, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vdiv_bu:

  case Intrinsic::loongarch_lsx_vdiv_hu:

  case Intrinsic::loongarch_lsx_vdiv_wu:

  case Intrinsic::loongarch_lsx_vdiv_du:

  case Intrinsic::loongarch_lasx_xvdiv_bu:

  case Intrinsic::loongarch_lasx_xvdiv_hu:

  case Intrinsic::loongarch_lasx_xvdiv_wu:

  case Intrinsic::loongarch_lasx_xvdiv_du:

    return DAG.getNode(ISD::UDIV, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmod_b:

  case Intrinsic::loongarch_lsx_vmod_h:

  case Intrinsic::loongarch_lsx_vmod_w:

  case Intrinsic::loongarch_lsx_vmod_d:

  case Intrinsic::loongarch_lasx_xvmod_b:

  case Intrinsic::loongarch_lasx_xvmod_h:

  case Intrinsic::loongarch_lasx_xvmod_w:

  case Intrinsic::loongarch_lasx_xvmod_d:

    return DAG.getNode(ISD::SREM, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vmod_bu:

  case Intrinsic::loongarch_lsx_vmod_hu:

  case Intrinsic::loongarch_lsx_vmod_wu:

  case Intrinsic::loongarch_lsx_vmod_du:

  case Intrinsic::loongarch_lasx_xvmod_bu:

  case Intrinsic::loongarch_lasx_xvmod_hu:

  case Intrinsic::loongarch_lasx_xvmod_wu:

  case Intrinsic::loongarch_lasx_xvmod_du:

    return DAG.getNode(ISD::UREM, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vand_v:

  case Intrinsic::loongarch_lasx_xvand_v:

    return DAG.getNode(ISD::AND, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vor_v:

  case Intrinsic::loongarch_lasx_xvor_v:

    return DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vxor_v:

  case Intrinsic::loongarch_lasx_xvxor_v:

    return DAG.getNode(ISD::XOR, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vnor_v:

  case Intrinsic::loongarch_lasx_xvnor_v: {

    SDValue Res = DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1),

                              N->getOperand(2));

    return DAG.getNOT(DL, Res, Res->getValueType(0));

  }

  case Intrinsic::loongarch_lsx_vandi_b:

  case Intrinsic::loongarch_lasx_xvandi_b:

    return DAG.getNode(ISD::AND, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<8>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vori_b:

  case Intrinsic::loongarch_lasx_xvori_b:

    return DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<8>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vxori_b:

  case Intrinsic::loongarch_lasx_xvxori_b:

    return DAG.getNode(ISD::XOR, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<8>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsll_b:

  case Intrinsic::loongarch_lsx_vsll_h:

  case Intrinsic::loongarch_lsx_vsll_w:

  case Intrinsic::loongarch_lsx_vsll_d:

  case Intrinsic::loongarch_lasx_xvsll_b:

  case Intrinsic::loongarch_lasx_xvsll_h:

  case Intrinsic::loongarch_lasx_xvsll_w:

  case Intrinsic::loongarch_lasx_xvsll_d:

    return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),

                       truncateVecElts(N, DAG));

  case Intrinsic::loongarch_lsx_vslli_b:

  case Intrinsic::loongarch_lasx_xvslli_b:

    return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<3>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vslli_h:

  case Intrinsic::loongarch_lasx_xvslli_h:

    return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<4>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vslli_w:

  case Intrinsic::loongarch_lasx_xvslli_w:

    return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vslli_d:

  case Intrinsic::loongarch_lasx_xvslli_d:

    return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<6>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrl_b:

  case Intrinsic::loongarch_lsx_vsrl_h:

  case Intrinsic::loongarch_lsx_vsrl_w:

  case Intrinsic::loongarch_lsx_vsrl_d:

  case Intrinsic::loongarch_lasx_xvsrl_b:

  case Intrinsic::loongarch_lasx_xvsrl_h:

  case Intrinsic::loongarch_lasx_xvsrl_w:

  case Intrinsic::loongarch_lasx_xvsrl_d:

    return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),

                       truncateVecElts(N, DAG));

  case Intrinsic::loongarch_lsx_vsrli_b:

  case Intrinsic::loongarch_lasx_xvsrli_b:

    return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<3>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrli_h:

  case Intrinsic::loongarch_lasx_xvsrli_h:

    return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<4>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrli_w:

  case Intrinsic::loongarch_lasx_xvsrli_w:

    return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrli_d:

  case Intrinsic::loongarch_lasx_xvsrli_d:

    return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<6>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsra_b:

  case Intrinsic::loongarch_lsx_vsra_h:

  case Intrinsic::loongarch_lsx_vsra_w:

  case Intrinsic::loongarch_lsx_vsra_d:

  case Intrinsic::loongarch_lasx_xvsra_b:

  case Intrinsic::loongarch_lasx_xvsra_h:

  case Intrinsic::loongarch_lasx_xvsra_w:

  case Intrinsic::loongarch_lasx_xvsra_d:

    return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),

                       truncateVecElts(N, DAG));

  case Intrinsic::loongarch_lsx_vsrai_b:

  case Intrinsic::loongarch_lasx_xvsrai_b:

    return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<3>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrai_h:

  case Intrinsic::loongarch_lasx_xvsrai_h:

    return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<4>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrai_w:

  case Intrinsic::loongarch_lasx_xvsrai_w:

    return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<5>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vsrai_d:

  case Intrinsic::loongarch_lasx_xvsrai_d:

    return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),

                       lowerVectorSplatImm<6>(N, 2, DAG));

  case Intrinsic::loongarch_lsx_vclz_b:

  case Intrinsic::loongarch_lsx_vclz_h:

  case Intrinsic::loongarch_lsx_vclz_w:

  case Intrinsic::loongarch_lsx_vclz_d:

  case Intrinsic::loongarch_lasx_xvclz_b:

  case Intrinsic::loongarch_lasx_xvclz_h:

  case Intrinsic::loongarch_lasx_xvclz_w:

  case Intrinsic::loongarch_lasx_xvclz_d:

    return DAG.getNode(ISD::CTLZ, DL, N->getValueType(0), N->getOperand(1));

  case Intrinsic::loongarch_lsx_vpcnt_b:

  case Intrinsic::loongarch_lsx_vpcnt_h:

  case Intrinsic::loongarch_lsx_vpcnt_w:

  case Intrinsic::loongarch_lsx_vpcnt_d:

  case Intrinsic::loongarch_lasx_xvpcnt_b:

  case Intrinsic::loongarch_lasx_xvpcnt_h:

  case Intrinsic::loongarch_lasx_xvpcnt_w:

  case Intrinsic::loongarch_lasx_xvpcnt_d:

    return DAG.getNode(ISD::CTPOP, DL, N->getValueType(0), N->getOperand(1));

  case Intrinsic::loongarch_lsx_vbitclr_b:

  case Intrinsic::loongarch_lsx_vbitclr_h:

  case Intrinsic::loongarch_lsx_vbitclr_w:

  case Intrinsic::loongarch_lsx_vbitclr_d:

  case Intrinsic::loongarch_lasx_xvbitclr_b:

  case Intrinsic::loongarch_lasx_xvbitclr_h:

  case Intrinsic::loongarch_lasx_xvbitclr_w:

  case Intrinsic::loongarch_lasx_xvbitclr_d:

    return lowerVectorBitClear(N, DAG);

  case Intrinsic::loongarch_lsx_vbitclri_b:

  case Intrinsic::loongarch_lasx_xvbitclri_b:

    return lowerVectorBitClearImm<3>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitclri_h:

  case Intrinsic::loongarch_lasx_xvbitclri_h:

    return lowerVectorBitClearImm<4>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitclri_w:

  case Intrinsic::loongarch_lasx_xvbitclri_w:

    return lowerVectorBitClearImm<5>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitclri_d:

  case Intrinsic::loongarch_lasx_xvbitclri_d:

    return lowerVectorBitClearImm<6>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitset_b:

  case Intrinsic::loongarch_lsx_vbitset_h:

  case Intrinsic::loongarch_lsx_vbitset_w:

  case Intrinsic::loongarch_lsx_vbitset_d:

  case Intrinsic::loongarch_lasx_xvbitset_b:

  case Intrinsic::loongarch_lasx_xvbitset_h:

  case Intrinsic::loongarch_lasx_xvbitset_w:

  case Intrinsic::loongarch_lasx_xvbitset_d: {

    EVT VecTy = N->getValueType(0);

    SDValue One = DAG.getConstant(1, DL, VecTy);

    return DAG.getNode(

        ISD::OR, DL, VecTy, N->getOperand(1),

        DAG.getNode(ISD::SHL, DL, VecTy, One, truncateVecElts(N, DAG)));

  }

  case Intrinsic::loongarch_lsx_vbitseti_b:

  case Intrinsic::loongarch_lasx_xvbitseti_b:

    return lowerVectorBitSetImm<3>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitseti_h:

  case Intrinsic::loongarch_lasx_xvbitseti_h:

    return lowerVectorBitSetImm<4>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitseti_w:

  case Intrinsic::loongarch_lasx_xvbitseti_w:

    return lowerVectorBitSetImm<5>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitseti_d:

  case Intrinsic::loongarch_lasx_xvbitseti_d:

    return lowerVectorBitSetImm<6>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitrev_b:

  case Intrinsic::loongarch_lsx_vbitrev_h:

  case Intrinsic::loongarch_lsx_vbitrev_w:

  case Intrinsic::loongarch_lsx_vbitrev_d:

  case Intrinsic::loongarch_lasx_xvbitrev_b:

  case Intrinsic::loongarch_lasx_xvbitrev_h:

  case Intrinsic::loongarch_lasx_xvbitrev_w:

  case Intrinsic::loongarch_lasx_xvbitrev_d: {

    EVT VecTy = N->getValueType(0);

    SDValue One = DAG.getConstant(1, DL, VecTy);

    return DAG.getNode(

        ISD::XOR, DL, VecTy, N->getOperand(1),

        DAG.getNode(ISD::SHL, DL, VecTy, One, truncateVecElts(N, DAG)));

  }

  case Intrinsic::loongarch_lsx_vbitrevi_b:

  case Intrinsic::loongarch_lasx_xvbitrevi_b:

    return lowerVectorBitRevImm<3>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitrevi_h:

  case Intrinsic::loongarch_lasx_xvbitrevi_h:

    return lowerVectorBitRevImm<4>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitrevi_w:

  case Intrinsic::loongarch_lasx_xvbitrevi_w:

    return lowerVectorBitRevImm<5>(N, DAG);

  case Intrinsic::loongarch_lsx_vbitrevi_d:

  case Intrinsic::loongarch_lasx_xvbitrevi_d:

    return lowerVectorBitRevImm<6>(N, DAG);

  case Intrinsic::loongarch_lsx_vfadd_s:

  case Intrinsic::loongarch_lsx_vfadd_d:

  case Intrinsic::loongarch_lasx_xvfadd_s:

  case Intrinsic::loongarch_lasx_xvfadd_d:

    return DAG.getNode(ISD::FADD, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vfsub_s:

  case Intrinsic::loongarch_lsx_vfsub_d:

  case Intrinsic::loongarch_lasx_xvfsub_s:

  case Intrinsic::loongarch_lasx_xvfsub_d:

    return DAG.getNode(ISD::FSUB, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vfmul_s:

  case Intrinsic::loongarch_lsx_vfmul_d:

  case Intrinsic::loongarch_lasx_xvfmul_s:

  case Intrinsic::loongarch_lasx_xvfmul_d:

    return DAG.getNode(ISD::FMUL, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vfdiv_s:

  case Intrinsic::loongarch_lsx_vfdiv_d:

  case Intrinsic::loongarch_lasx_xvfdiv_s:

  case Intrinsic::loongarch_lasx_xvfdiv_d:

    return DAG.getNode(ISD::FDIV, DL, N->getValueType(0), N->getOperand(1),

                       N->getOperand(2));

  case Intrinsic::loongarch_lsx_vfmadd_s:

  case Intrinsic::loongarch_lsx_vfmadd_d:

</