/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

Bug Summary

File:	llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
Warning:	line 2112, column 9 The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long'

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name WebAssemblyISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly -I include -I /build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/build-llvm -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-10-18-030439-21163-1 -x c++ /build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

→

1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//

14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "Utils/WebAssemblyTypeUtilities.h"
17#include "Utils/WebAssemblyUtilities.h"
18#include "WebAssemblyMachineFunctionInfo.h"
19#include "WebAssemblySubtarget.h"
20#include "WebAssemblyTargetMachine.h"
21#include "llvm/CodeGen/CallingConvLower.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineModuleInfo.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/DiagnosticPrinter.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/IntrinsicsWebAssembly.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Target/TargetOptions.h"
39using namespace llvm;

41#define DEBUG_TYPE"wasm-lower" "wasm-lower"

43WebAssemblyTargetLowering::WebAssemblyTargetLowering(
  const TargetMachine &TM, const WebAssemblySubtarget &STI)
  : TargetLowering(TM), Subtarget(&STI) {
auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;

// Booleans always contain 0 or 1.
setBooleanContents(ZeroOrOneBooleanContent);
// Except in SIMD vectors
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
// We don't know the microarchitecture here, so just reduce register pressure.
setSchedulingPreference(Sched::RegPressure);
// Tell ISel that we have a stack pointer.
setStackPointerRegisterToSaveRestore(
    Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
// Set up the register classes.
addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
if (Subtarget->hasSIMD128()) {
  addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
}
if (Subtarget->hasReferenceTypes()) {
  addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
  addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
}
// Compute derived properties from the register classes.
computeRegisterProperties(Subtarget->getRegisterInfo());

// Transform loads and stores to pointers in address space 1 to loads and
// stores to WebAssembly global variables, outside linear memory.
for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
  setOperationAction(ISD::LOAD, T, Custom);
  setOperationAction(ISD::STORE, T, Custom);
}
if (Subtarget->hasSIMD128()) {
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64}) {
    setOperationAction(ISD::LOAD, T, Custom);
    setOperationAction(ISD::STORE, T, Custom);
  }
}
if (Subtarget->hasReferenceTypes()) {
  for (auto T : {MVT::externref, MVT::funcref}) {
    setOperationAction(ISD::LOAD, T, Custom);
    setOperationAction(ISD::STORE, T, Custom);
  }
}

setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom);
setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
setOperationAction(ISD::JumpTable, MVTPtr, Custom);
setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
setOperationAction(ISD::BRIND, MVT::Other, Custom);

// Take the default expansion for va_arg, va_copy, and va_end. There is no
// default action for va_start, so we do that custom.
setOperationAction(ISD::VASTART, MVT::Other, Custom);
setOperationAction(ISD::VAARG, MVT::Other, Expand);
setOperationAction(ISD::VACOPY, MVT::Other, Expand);
setOperationAction(ISD::VAEND, MVT::Other, Expand);

for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
  // Don't expand the floating-point types to constant pools.
  setOperationAction(ISD::ConstantFP, T, Legal);
  // Expand floating-point comparisons.
  for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
                  ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
    setCondCodeAction(CC, T, Expand);
  // Expand floating-point library function operators.
  for (auto Op :
       {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
    setOperationAction(Op, T, Expand);
  // Note supported floating-point library function operators that otherwise
  // default to expand.
  for (auto Op :
       {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
    setOperationAction(Op, T, Legal);
  // Support minimum and maximum, which otherwise default to expand.
  setOperationAction(ISD::FMINIMUM, T, Legal);
  setOperationAction(ISD::FMAXIMUM, T, Legal);
  // WebAssembly currently has no builtin f16 support.
  setOperationAction(ISD::FP16_TO_FP, T, Expand);
  setOperationAction(ISD::FP_TO_FP16, T, Expand);
  setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
  setTruncStoreAction(T, MVT::f16, Expand);
}

// Expand unavailable integer operations.
for (auto Op :
     {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
      ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
      ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
  for (auto T : {MVT::i32, MVT::i64})
    setOperationAction(Op, T, Expand);
  if (Subtarget->hasSIMD128())
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(Op, T, Expand);
}

if (Subtarget->hasNontrappingFPToInt())
  for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
    for (auto T : {MVT::i32, MVT::i64})
      setOperationAction(Op, T, Custom);

// SIMD-specific configuration
if (Subtarget->hasSIMD128()) {
  // Hoist bitcasts out of shuffles
  setTargetDAGCombine(ISD::VECTOR_SHUFFLE);

  // Combine extends of extract_subvectors into widening ops
  setTargetDAGCombine(ISD::SIGN_EXTEND);
  setTargetDAGCombine(ISD::ZERO_EXTEND);

  // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
  // conversions ops
  setTargetDAGCombine(ISD::SINT_TO_FP);
  setTargetDAGCombine(ISD::UINT_TO_FP);
  setTargetDAGCombine(ISD::FP_EXTEND);
  setTargetDAGCombine(ISD::EXTRACT_SUBVECTOR);

  // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
  // into conversion ops
  setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
  setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
  setTargetDAGCombine(ISD::FP_ROUND);
  setTargetDAGCombine(ISD::CONCAT_VECTORS);

  // Support saturating add for i8x16 and i16x8
  for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
    for (auto T : {MVT::v16i8, MVT::v8i16})
      setOperationAction(Op, T, Legal);

  // Support integer abs
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
    setOperationAction(ISD::ABS, T, Legal);

  // Custom lower BUILD_VECTORs to minimize number of replace_lanes
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64})
    setOperationAction(ISD::BUILD_VECTOR, T, Custom);

  // We have custom shuffle lowering to expose the shuffle mask
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64})
    setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);

  // Custom lowering since wasm shifts must have a scalar shift amount
  for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(Op, T, Custom);

  // Custom lower lane accesses to expand out variable indices
  for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                   MVT::v2f64})
      setOperationAction(Op, T, Custom);

  // There is no i8x16.mul instruction
  setOperationAction(ISD::MUL, MVT::v16i8, Expand);

  // There is no vector conditional select instruction
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64})
    setOperationAction(ISD::SELECT_CC, T, Expand);

  // Expand integer operations supported for scalars but not SIMD
  for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV,
                  ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(Op, T, Expand);

  // But we do have integer min and max operations
  for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
      setOperationAction(Op, T, Legal);

  // And we have popcnt for i8x16
  setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);

  // Expand float operations supported for scalars but not SIMD
  for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
                  ISD::FEXP, ISD::FEXP2, ISD::FRINT})
    for (auto T : {MVT::v4f32, MVT::v2f64})
      setOperationAction(Op, T, Expand);

  // Unsigned comparison operations are unavailable for i64x2 vectors.
  for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
    setCondCodeAction(CC, MVT::v2i64, Custom);

  // 64x2 conversions are not in the spec
  for (auto Op :
       {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
    for (auto T : {MVT::v2i64, MVT::v2f64})
      setOperationAction(Op, T, Expand);

  // But saturating fp_to_int converstions are
  for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
    setOperationAction(Op, MVT::v4i32, Custom);
}

// As a special case, these operators use the type to mean the type to
// sign-extend from.
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
if (!Subtarget->hasSignExt()) {
  // Sign extends are legal only when extending a vector extract
  auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
  for (auto T : {MVT::i8, MVT::i16, MVT::i32})
    setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
}
for (auto T : MVT::integer_fixedlen_vector_valuetypes())
  setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);

// Dynamic stack allocation: use the default expansion.
setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);

setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
setOperationAction(ISD::FrameIndex, MVT::i64, Custom);
setOperationAction(ISD::CopyToReg, MVT::Other, Custom);

// Expand these forms; we pattern-match the forms that we can handle in isel.
for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
  for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
    setOperationAction(Op, T, Expand);

// We have custom switch handling.
setOperationAction(ISD::BR_JT, MVT::Other, Custom);

// WebAssembly doesn't have:
//  - Floating-point extending loads.
//  - Floating-point truncating stores.
//  - i1 extending loads.
//  - truncating SIMD stores and most extending loads
setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
setTruncStoreAction(MVT::f64, MVT::f32, Expand);
for (auto T : MVT::integer_valuetypes())
  for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
    setLoadExtAction(Ext, T, MVT::i1, Promote);
if (Subtarget->hasSIMD128()) {
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
                 MVT::v2f64}) {
    for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
      if (MVT(T) != MemT) {
        setTruncStoreAction(T, MemT, Expand);
        for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
          setLoadExtAction(Ext, T, MemT, Expand);
      }
    }
  }
  // But some vector extending loads are legal
  for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
    setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
    setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
    setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
  }
  setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);
}

// Don't do anything clever with build_pairs
setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);

// Trap lowers to wasm unreachable
setOperationAction(ISD::TRAP, MVT::Other, Legal);
setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);

// Exception handling intrinsics
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);

setMaxAtomicSizeInBitsSupported(64);

// Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
// consistent with the f64 and f128 names.
setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");

// Define the emscripten name for return address helper.
// TODO: when implementing other Wasm backends, make this generic or only do
// this on emscripten depending on what they end up doing.
setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");

// Always convert switches to br_tables unless there is only one case, which
// is equivalent to a simple branch. This reduces code size for wasm, and we
// defer possible jump table optimizations to the VM.
setMinimumJumpTableEntries(2);
337}

339MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL,
                                          uint32_t AS) const {
if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
  return MVT::externref;
if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
  return MVT::funcref;
return TargetLowering::getPointerTy(DL, AS);
346}

348MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,
                                             uint32_t AS) const {
if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
  return MVT::externref;
if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
  return MVT::funcref;
return TargetLowering::getPointerMemTy(DL, AS);
355}

357TargetLowering::AtomicExpansionKind
358WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// We have wasm instructions for these
switch (AI->getOperation()) {
case AtomicRMWInst::Add:
case AtomicRMWInst::Sub:
case AtomicRMWInst::And:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
case AtomicRMWInst::Xchg:
  return AtomicExpansionKind::None;
default:
  break;
}
return AtomicExpansionKind::CmpXChg;
372}

374bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
// Implementation copied from X86TargetLowering.
unsigned Opc = VecOp.getOpcode();

// Assume target opcodes can't be scalarized.
// TODO - do we have any exceptions?
if (Opc >= ISD::BUILTIN_OP_END)
  return false;

// If the vector op is not supported, try to convert to scalar.
EVT VecVT = VecOp.getValueType();
if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
  return true;

// If the vector op is supported, but the scalar op is not, the transform may
// not be worthwhile.
EVT ScalarVT = VecVT.getScalarType();
return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
392}

394FastISel *WebAssemblyTargetLowering::createFastISel(
  FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
return WebAssembly::createFastISel(FuncInfo, LibInfo);
397}

399MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
                                                    EVT VT) const {
unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
if (BitWidth > 1 && BitWidth < 8)
  BitWidth = 8;

if (BitWidth > 64) {
  // The shift will be lowered to a libcall, and compiler-rt libcalls expect
  // the count to be an i32.
  BitWidth = 32;
  assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 410, __extension__ __PRETTY_FUNCTION__))
         "32-bit shift counts ought to be enough for anyone")(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 410, __extension__ __PRETTY_FUNCTION__));
}

MVT Result = MVT::getIntegerVT(BitWidth);
assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
 && "Unable to represent scalar shift amount type") ?
 void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 415, __extension__ __PRETTY_FUNCTION__))
       "Unable to represent scalar shift amount type")(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
 && "Unable to represent scalar shift amount type") ?
 void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 415, __extension__ __PRETTY_FUNCTION__));
return Result;
417}

419// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
420// undefined result on invalid/overflow, to the WebAssembly opcode, which
421// traps on invalid/overflow.
422static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
                                     MachineBasicBlock *BB,
                                     const TargetInstrInfo &TII,
                                     bool IsUnsigned, bool Int64,
                                     bool Float64, unsigned LoweredOpcode) {
MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();

Register OutReg = MI.getOperand(0).getReg();
Register InReg = MI.getOperand(1).getReg();

unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
unsigned Eqz = WebAssembly::EQZ_I32;
unsigned And = WebAssembly::AND_I32;
int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1);
int64_t Substitute = IsUnsigned ? 0 : Limit;
double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
auto &Context = BB->getParent()->getFunction().getContext();
Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);

const BasicBlock *LLVMBB = BB->getBasicBlock();
MachineFunction *F = BB->getParent();
MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);

MachineFunction::iterator It = ++BB->getIterator();
F->insert(It, FalseMBB);
F->insert(It, TrueMBB);
F->insert(It, DoneMBB);

// Transfer the remainder of BB and its successor edges to DoneMBB.
DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
DoneMBB->transferSuccessorsAndUpdatePHIs(BB);

BB->addSuccessor(TrueMBB);
BB->addSuccessor(FalseMBB);
TrueMBB->addSuccessor(DoneMBB);
FalseMBB->addSuccessor(DoneMBB);

unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));

MI.eraseFromParent();
// For signed numbers, we can do a single comparison to determine whether
// fabs(x) is within range.
if (IsUnsigned) {
  Tmp0 = InReg;
} else {
  BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
}
BuildMI(BB, DL, TII.get(FConst), Tmp1)
    .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);

// For unsigned numbers, we have to do a separate comparison with zero.
if (IsUnsigned) {
  Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
  Register SecondCmpReg =
      MRI.createVirtualRegister(&WebAssembly::I32RegClass);
  Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
  BuildMI(BB, DL, TII.get(FConst), Tmp1)
      .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
  BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
  BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
  CmpReg = AndReg;
}

BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);

// Create the CFG diamond to select between doing the conversion or using
// the substitute value.
BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
    .addReg(FalseReg)
    .addMBB(FalseMBB)
    .addReg(TrueReg)
    .addMBB(TrueMBB);

return DoneMBB;
513}

515static MachineBasicBlock *
516LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
               const WebAssemblySubtarget *Subtarget,
               const TargetInstrInfo &TII) {
MachineInstr &CallParams = *CallResults.getPrevNode();
assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS)(static_cast <bool> (CallParams.getOpcode() == WebAssembly
::CALL_PARAMS) ? void (0) : __assert_fail ("CallParams.getOpcode() == WebAssembly::CALL_PARAMS"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 520, __extension__ __PRETTY_FUNCTION__));
assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 522, __extension__ __PRETTY_FUNCTION__))
       CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS)(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 522, __extension__ __PRETTY_FUNCTION__));

bool IsIndirect = CallParams.getOperand(0).isReg();
bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;

bool IsFuncrefCall = false;
if (IsIndirect) {
  Register Reg = CallParams.getOperand(0).getReg();
  const MachineFunction *MF = BB->getParent();
  const MachineRegisterInfo &MRI = MF->getRegInfo();
  const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
  IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
  assert(!IsFuncrefCall || Subtarget->hasReferenceTypes())(static_cast <bool> (!IsFuncrefCall || Subtarget->hasReferenceTypes
()) ? void (0) : __assert_fail ("!IsFuncrefCall || Subtarget->hasReferenceTypes()"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 534, __extension__ __PRETTY_FUNCTION__));
}

unsigned CallOp;
if (IsIndirect && IsRetCall) {
  CallOp = WebAssembly::RET_CALL_INDIRECT;
} else if (IsIndirect) {
  CallOp = WebAssembly::CALL_INDIRECT;
} else if (IsRetCall) {
  CallOp = WebAssembly::RET_CALL;
} else {
  CallOp = WebAssembly::CALL;
}

MachineFunction &MF = *BB->getParent();
const MCInstrDesc &MCID = TII.get(CallOp);
MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));

// See if we must truncate the function pointer.
// CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers
// as 64-bit for uniformity with other pointer types.
// See also: WebAssemblyFastISel::selectCall
if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) {
  Register Reg32 =
      MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
  auto &FnPtr = CallParams.getOperand(0);
  BuildMI(*BB, CallResults.getIterator(), DL,
          TII.get(WebAssembly::I32_WRAP_I64), Reg32)
      .addReg(FnPtr.getReg());
  FnPtr.setReg(Reg32);
}

// Move the function pointer to the end of the arguments for indirect calls
if (IsIndirect) {
  auto FnPtr = CallParams.getOperand(0);
  CallParams.RemoveOperand(0);

  // For funcrefs, call_indirect is done through __funcref_call_table and the
  // funcref is always installed in slot 0 of the table, therefore instead of having
  // the function pointer added at the end of the params list, a zero (the index in
  // __funcref_call_table is added).
  if (IsFuncrefCall) {
    Register RegZero =
        MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
    MachineInstrBuilder MIBC0 =
        BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);

    BB->insert(CallResults.getIterator(), MIBC0);
    MachineInstrBuilder(MF, CallParams).addReg(RegZero);
  } else
    CallParams.addOperand(FnPtr);
}

for (auto Def : CallResults.defs())
  MIB.add(Def);

if (IsIndirect) {
  // Placeholder for the type index.
  MIB.addImm(0);
  // The table into which this call_indirect indexes.
  MCSymbolWasm *Table = IsFuncrefCall
                            ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
                                  MF.getContext(), Subtarget)
                            : WebAssembly::getOrCreateFunctionTableSymbol(
                                  MF.getContext(), Subtarget);
  if (Subtarget->hasReferenceTypes()) {
    MIB.addSym(Table);
  } else {
    // For the MVP there is at most one table whose number is 0, but we can't
    // write a table symbol or issue relocations.  Instead we just ensure the
    // table is live and write a zero.
    Table->setNoStrip();
    MIB.addImm(0);
  }
}

for (auto Use : CallParams.uses())
  MIB.add(Use);

BB->insert(CallResults.getIterator(), MIB);
CallParams.eraseFromParent();
CallResults.eraseFromParent();

// If this is a funcref call, to avoid hidden GC roots, we need to clear the
// table slot with ref.null upon call_indirect return.
//
// This generates the following code, which comes right after a call_indirect
// of a funcref:
//
//    i32.const 0
//    ref.null func
//    table.set __funcref_call_table
if (IsIndirect && IsFuncrefCall) {
  MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
      MF.getContext(), Subtarget);
  Register RegZero =
      MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
  MachineInstr *Const0 =
      BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
  BB->insertAfter(MIB.getInstr()->getIterator(), Const0);

  Register RegFuncref =
      MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
  MachineInstr *RefNull =
      BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref)
          .addImm(static_cast<int32_t>(WebAssembly::HeapType::Funcref));
  BB->insertAfter(Const0->getIterator(), RefNull);

  MachineInstr *TableSet =
      BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
          .addSym(Table)
          .addReg(RegZero)
          .addReg(RegFuncref);
  BB->insertAfter(RefNull->getIterator(), TableSet);
}

return BB;
651}

653MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
  MachineInstr &MI, MachineBasicBlock *BB) const {
const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
DebugLoc DL = MI.getDebugLoc();

switch (MI.getOpcode()) {
default:
  llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 660);
case WebAssembly::FP_TO_SINT_I32_F32:
  return LowerFPToInt(MI, DL, BB, TII, false, false, false,
                      WebAssembly::I32_TRUNC_S_F32);
case WebAssembly::FP_TO_UINT_I32_F32:
  return LowerFPToInt(MI, DL, BB, TII, true, false, false,
                      WebAssembly::I32_TRUNC_U_F32);
case WebAssembly::FP_TO_SINT_I64_F32:
  return LowerFPToInt(MI, DL, BB, TII, false, true, false,
                      WebAssembly::I64_TRUNC_S_F32);
case WebAssembly::FP_TO_UINT_I64_F32:
  return LowerFPToInt(MI, DL, BB, TII, true, true, false,
                      WebAssembly::I64_TRUNC_U_F32);
case WebAssembly::FP_TO_SINT_I32_F64:
  return LowerFPToInt(MI, DL, BB, TII, false, false, true,
                      WebAssembly::I32_TRUNC_S_F64);
case WebAssembly::FP_TO_UINT_I32_F64:
  return LowerFPToInt(MI, DL, BB, TII, true, false, true,
                      WebAssembly::I32_TRUNC_U_F64);
case WebAssembly::FP_TO_SINT_I64_F64:
  return LowerFPToInt(MI, DL, BB, TII, false, true, true,
                      WebAssembly::I64_TRUNC_S_F64);
case WebAssembly::FP_TO_UINT_I64_F64:
  return LowerFPToInt(MI, DL, BB, TII, true, true, true,
                      WebAssembly::I64_TRUNC_U_F64);
case WebAssembly::CALL_RESULTS:
case WebAssembly::RET_CALL_RESULTS:
  return LowerCallResults(MI, DL, BB, Subtarget, TII);
}
689}

691const char *
692WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
case WebAssemblyISD::FIRST_NUMBER:
case WebAssemblyISD::FIRST_MEM_OPCODE:
  break;
697#define HANDLE_NODETYPE(NODE)                                                  \
case WebAssemblyISD::NODE:                                                   \
  return "WebAssemblyISD::" #NODE;
700#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
701#include "WebAssemblyISD.def"
702#undef HANDLE_MEM_NODETYPE
703#undef HANDLE_NODETYPE
}
return nullptr;
706}

708std::pair<unsigned, const TargetRegisterClass *>
709WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
  const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
// First, see if this is a constraint that directly corresponds to a
// WebAssembly register class.
if (Constraint.size() == 1) {
  switch (Constraint[0]) {
  case 'r':
    assert(VT != MVT::iPTR && "Pointer MVT not expected here")(static_cast <bool> (VT != MVT::iPTR && "Pointer MVT not expected here"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 716, __extension__ __PRETTY_FUNCTION__));
    if (Subtarget->hasSIMD128() && VT.isVector()) {
      if (VT.getSizeInBits() == 128)
        return std::make_pair(0U, &WebAssembly::V128RegClass);
    }
    if (VT.isInteger() && !VT.isVector()) {
      if (VT.getSizeInBits() <= 32)
        return std::make_pair(0U, &WebAssembly::I32RegClass);
      if (VT.getSizeInBits() <= 64)
        return std::make_pair(0U, &WebAssembly::I64RegClass);
    }
    if (VT.isFloatingPoint() && !VT.isVector()) {
      switch (VT.getSizeInBits()) {
      case 32:
        return std::make_pair(0U, &WebAssembly::F32RegClass);
      case 64:
        return std::make_pair(0U, &WebAssembly::F64RegClass);
      default:
        break;
      }
    }
    break;
  default:
    break;
  }
}

return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
744}

746bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const {
// Assume ctz is a relatively cheap operation.
return true;
749}

751bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
// Assume clz is a relatively cheap operation.
return true;
754}

756bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                                    const AddrMode &AM,
                                                    Type *Ty, unsigned AS,
                                                    Instruction *I) const {
// WebAssembly offsets are added as unsigned without wrapping. The
// isLegalAddressingMode gives us no way to determine if wrapping could be
// happening, so we approximate this by accepting only non-negative offsets.
if (AM.BaseOffs < 0)
  return false;

// WebAssembly has no scale register operands.
if (AM.Scale != 0)
  return false;

// Everything else is legal.
return true;
772}

774bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
  EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
  MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
// WebAssembly supports unaligned accesses, though it should be declared
// with the p2align attribute on loads and stores which do so, and there
// may be a performance impact. We tell LLVM they're "fast" because
// for the kinds of things that LLVM uses this for (merging adjacent stores
// of constants, etc.), WebAssembly implementations will either want the
// unaligned access or they'll split anyway.
if (Fast)
  *Fast = true;
return true;
786}

788bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
                                            AttributeList Attr) const {
// The current thinking is that wasm engines will perform this optimization,
// so we can save on code size.
return true;
793}

795bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
EVT ExtT = ExtVal.getValueType();
EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
       (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
       (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
801}

803bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
  const GlobalAddressSDNode *GA) const {
// Wasm doesn't support function addresses with offsets
const GlobalValue *GV = GA->getGlobal();
return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
808}

810EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
                                                LLVMContext &C,
                                                EVT VT) const {
if (VT.isVector())
  return VT.changeVectorElementTypeToInteger();

// So far, all branch instructions in Wasm take an I32 condition.
// The default TargetLowering::getSetCCResultType returns the pointer size,
// which would be useful to reduce instruction counts when testing
// against 64-bit pointers/values if at some point Wasm supports that.
return EVT::getIntegerVT(C, 32);
821}

823bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                                 const CallInst &I,
                                                 MachineFunction &MF,
                                                 unsigned Intrinsic) const {
switch (Intrinsic) {
case Intrinsic::wasm_memory_atomic_notify:
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::i32;
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(4);
  // atomic.notify instruction does not really load the memory specified with
  // this argument, but MachineMemOperand should either be load or store, so
  // we set this to a load.
  // FIXME Volatile isn't really correct, but currently all LLVM atomic
  // instructions are treated as volatiles in the backend, so we should be
  // consistent. The same applies for wasm_atomic_wait intrinsics too.
  Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
  return true;
case Intrinsic::wasm_memory_atomic_wait32:
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::i32;
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(4);
  Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
  return true;
case Intrinsic::wasm_memory_atomic_wait64:
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::i64;
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(8);
  Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
  return true;
default:
  return false;
}
861}

863void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
  const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
  const SelectionDAG &DAG, unsigned Depth) const {
switch (Op.getOpcode()) {
default:
  break;
case ISD::INTRINSIC_WO_CHAIN: {
  unsigned IntNo = Op.getConstantOperandVal(0);
  switch (IntNo) {
  default:
    break;
  case Intrinsic::wasm_bitmask: {
    unsigned BitWidth = Known.getBitWidth();
    EVT VT = Op.getOperand(1).getSimpleValueType();
    unsigned PossibleBits = VT.getVectorNumElements();
    APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
    Known.Zero |= ZeroMask;
    break;
  }
  }
}
}
885}

887TargetLoweringBase::LegalizeTypeAction
888WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
if (VT.isFixedLengthVector()) {
  MVT EltVT = VT.getVectorElementType();
  // We have legal vector types with these lane types, so widening the
  // vector would let us use some of the lanes directly without having to
  // extend or truncate values.
  if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
      EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
    return TypeWidenVector;
}

return TargetLoweringBase::getPreferredVectorAction(VT);
900}

902//===----------------------------------------------------------------------===//
903// WebAssembly Lowering private implementation.
904//===----------------------------------------------------------------------===//

906//===----------------------------------------------------------------------===//
907// Lowering Code
908//===----------------------------------------------------------------------===//

910static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
MachineFunction &MF = DAG.getMachineFunction();
DAG.getContext()->diagnose(
    DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
914}

916// Test whether the given calling convention is supported.
917static bool callingConvSupported(CallingConv::ID CallConv) {
// We currently support the language-independent target-independent
// conventions. We don't yet have a way to annotate calls with properties like
// "cold", and we don't have any call-clobbered registers, so these are mostly
// all handled the same.
return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
       CallConv == CallingConv::Cold ||
       CallConv == CallingConv::PreserveMost ||
       CallConv == CallingConv::PreserveAll ||
       CallConv == CallingConv::CXX_FAST_TLS ||
       CallConv == CallingConv::WASM_EmscriptenInvoke ||
       CallConv == CallingConv::Swift;
929}

931SDValue
932WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
                                   SmallVectorImpl<SDValue> &InVals) const {
SelectionDAG &DAG = CLI.DAG;
SDLoc DL = CLI.DL;
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
MachineFunction &MF = DAG.getMachineFunction();
auto Layout = MF.getDataLayout();

CallingConv::ID CallConv = CLI.CallConv;
if (!callingConvSupported(CallConv))
  fail(DL, DAG,
       "WebAssembly doesn't support language-specific or target-specific "
       "calling conventions yet");
if (CLI.IsPatchPoint)
  fail(DL, DAG, "WebAssembly doesn't support patch point yet");

if (CLI.IsTailCall) {
  auto NoTail = [&](const char *Msg) {
    if (CLI.CB && CLI.CB->isMustTailCall())
      fail(DL, DAG, Msg);
    CLI.IsTailCall = false;
  };

  if (!Subtarget->hasTailCall())
    NoTail("WebAssembly 'tail-call' feature not enabled");

  // Varargs calls cannot be tail calls because the buffer is on the stack
  if (CLI.IsVarArg)
    NoTail("WebAssembly does not support varargs tail calls");

  // Do not tail call unless caller and callee return types match
  const Function &F = MF.getFunction();
  const TargetMachine &TM = getTargetMachine();
  Type *RetTy = F.getReturnType();
  SmallVector<MVT, 4> CallerRetTys;
  SmallVector<MVT, 4> CalleeRetTys;
  computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
  computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
  bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
                    std::equal(CallerRetTys.begin(), CallerRetTys.end(),
                               CalleeRetTys.begin());
  if (!TypesMatch)
    NoTail("WebAssembly tail call requires caller and callee return types to "
           "match");

  // If pointers to local stack values are passed, we cannot tail call
  if (CLI.CB) {
    for (auto &Arg : CLI.CB->args()) {
      Value *Val = Arg.get();
      // Trace the value back through pointer operations
      while (true) {
        Value *Src = Val->stripPointerCastsAndAliases();
        if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
          Src = GEP->getPointerOperand();
        if (Val == Src)
          break;
        Val = Src;
      }
      if (isa<AllocaInst>(Val)) {
        NoTail(
            "WebAssembly does not support tail calling with stack arguments");
        break;
      }
    }
  }
}

SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;

// The generic code may have added an sret argument. If we're lowering an
// invoke function, the ABI requires that the function pointer be the first
// argument, so we may have to swap the arguments.
if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
    Outs[0].Flags.isSRet()) {
  std::swap(Outs[0], Outs[1]);
  std::swap(OutVals[0], OutVals[1]);
}

bool HasSwiftSelfArg = false;
bool HasSwiftErrorArg = false;
unsigned NumFixedArgs = 0;
for (unsigned I = 0; I < Outs.size(); ++I) {
  const ISD::OutputArg &Out = Outs[I];
  SDValue &OutVal = OutVals[I];
  HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
  HasSwiftErrorArg |= Out.Flags.isSwiftError();
  if (Out.Flags.isNest())
    fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
  if (Out.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
  if (Out.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
  if (Out.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
  if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
    auto &MFI = MF.getFrameInfo();
    int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
                                   Out.Flags.getNonZeroByValAlign(),
                                   /*isSS=*/false);
    SDValue SizeNode =
        DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
    SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
    Chain = DAG.getMemcpy(
        Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
        /*isVolatile*/ false, /*AlwaysInline=*/false,
        /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
    OutVal = FINode;
  }
  // Count the number of fixed args *after* legalization.
  NumFixedArgs += Out.IsFixed;
}

bool IsVarArg = CLI.IsVarArg;
auto PtrVT = getPointerTy(Layout);

// For swiftcc, emit additional swiftself and swifterror arguments
// if there aren't. These additional arguments are also added for callee
// signature They are necessary to match callee and caller signature for
// indirect call.
if (CallConv == CallingConv::Swift) {
  if (!HasSwiftSelfArg) {
    NumFixedArgs++;
    ISD::OutputArg Arg;
    Arg.Flags.setSwiftSelf();
    CLI.Outs.push_back(Arg);
    SDValue ArgVal = DAG.getUNDEF(PtrVT);
    CLI.OutVals.push_back(ArgVal);
  }
  if (!HasSwiftErrorArg) {
    NumFixedArgs++;
    ISD::OutputArg Arg;
    Arg.Flags.setSwiftError();
    CLI.Outs.push_back(Arg);
    SDValue ArgVal = DAG.getUNDEF(PtrVT);
    CLI.OutVals.push_back(ArgVal);
  }
}

// Analyze operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());

if (IsVarArg) {
  // Outgoing non-fixed arguments are placed in a buffer. First
  // compute their offsets and the total amount of buffer space needed.
  for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
    const ISD::OutputArg &Out = Outs[I];
    SDValue &Arg = OutVals[I];
    EVT VT = Arg.getValueType();
    assert(VT != MVT::iPTR && "Legalized args should be concrete")(static_cast <bool> (VT != MVT::iPTR && "Legalized args should be concrete"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1084, __extension__ __PRETTY_FUNCTION__));
    Type *Ty = VT.getTypeForEVT(*DAG.getContext());
    Align Alignment =
        std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
    unsigned Offset =
        CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
    CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
                                      Offset, VT.getSimpleVT(),
                                      CCValAssign::Full));
  }
}

unsigned NumBytes = CCInfo.getAlignedCallFrameSize();

SDValue FINode;
if (IsVarArg && NumBytes) {
  // For non-fixed arguments, next emit stores to store the argument values
  // to the stack buffer at the offsets computed above.
  int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
                                               Layout.getStackAlignment(),
                                               /*isSS=*/false);
  unsigned ValNo = 0;
  SmallVector<SDValue, 8> Chains;
  for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
    assert(ArgLocs[ValNo].getValNo() == ValNo &&(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
 && "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1109, __extension__ __PRETTY_FUNCTION__))
           "ArgLocs should remain in order and only hold varargs args")(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
 && "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1109, __extension__ __PRETTY_FUNCTION__));
    unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
    FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
    SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
                              DAG.getConstant(Offset, DL, PtrVT));
    Chains.push_back(
        DAG.getStore(Chain, DL, Arg, Add,
                     MachinePointerInfo::getFixedStack(MF, FI, Offset)));
  }
  if (!Chains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
} else if (IsVarArg) {
  FINode = DAG.getIntPtrConstant(0, DL);
}

if (Callee->getOpcode() == ISD::GlobalAddress) {
  // If the callee is a GlobalAddress node (quite common, every direct call
  // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
  // doesn't at MO_GOT which is not needed for direct calls.
  GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee);
  Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
                                      getPointerTy(DAG.getDataLayout()),
                                      GA->getOffset());
  Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
                       getPointerTy(DAG.getDataLayout()), Callee);
}

// Compute the operands for the CALLn node.
SmallVector<SDValue, 16> Ops;
Ops.push_back(Chain);
Ops.push_back(Callee);

// Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
// isn't reliable.
Ops.append(OutVals.begin(),
           IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
// Add a pointer to the vararg buffer.
if (IsVarArg)
  Ops.push_back(FINode);

SmallVector<EVT, 8> InTys;
for (const auto &In : Ins) {
  assert(!In.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!In.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1151, __extension__ __PRETTY_FUNCTION__));
  assert(!In.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!In.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1152, __extension__ __PRETTY_FUNCTION__));
  if (In.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
  if (In.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
  if (In.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG,
         "WebAssembly hasn't implemented cons regs last return values");
  // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
  // registers.
  InTys.push_back(In.VT);
}

// Lastly, if this is a call to a funcref we need to add an instruction
// table.set to the chain and transform the call.
if (CLI.CB &&
    WebAssembly::isFuncrefType(CLI.CB->getCalledOperand()->getType())) {
  // In the absence of function references proposal where a funcref call is
  // lowered to call_ref, using reference types we generate a table.set to set
  // the funcref to a special table used solely for this purpose, followed by
  // a call_indirect. Here we just generate the table set, and return the
  // SDValue of the table.set so that LowerCall can finalize the lowering by
  // generating the call_indirect.
  SDValue Chain = Ops[0];

  MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
      MF.getContext(), Subtarget);
  SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
  SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
  SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
  SDValue TableSet = DAG.getMemIntrinsicNode(
      WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
      MVT::funcref,
      // Machine Mem Operand args
      MachinePointerInfo(
          WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF),
      CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
      MachineMemOperand::MOStore);

  Ops[0] = TableSet; // The new chain is the TableSet itself
}

if (CLI.IsTailCall) {
  // ret_calls do not return values to the current frame
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
  return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
}

InTys.push_back(MVT::Other);
SDVTList InTyList = DAG.getVTList(InTys);
SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);

for (size_t I = 0; I < Ins.size(); ++I)
  InVals.push_back(Res.getValue(I));

// Return the chain
return Res.getValue(Ins.size());
1209}

1211bool WebAssemblyTargetLowering::CanLowerReturn(
  CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
  const SmallVectorImpl<ISD::OutputArg> &Outs,
  LLVMContext & /*Context*/) const {
// WebAssembly can only handle returning tuples with multivalue enabled
return Subtarget->hasMultivalue() || Outs.size() <= 1;
1217}

1219SDValue WebAssemblyTargetLowering::LowerReturn(
  SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
  const SmallVectorImpl<ISD::OutputArg> &Outs,
  const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
  SelectionDAG &DAG) const {
assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__))
       "MVP WebAssembly can only return up to one value")(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__));
if (!callingConvSupported(CallConv))
  fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");

SmallVector<SDValue, 4> RetOps(1, Chain);
RetOps.append(OutVals.begin(), OutVals.end());
Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);

// Record the number and types of the return values.
for (const ISD::OutputArg &Out : Outs) {
  assert(!Out.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!Out.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1235, __extension__ __PRETTY_FUNCTION__));
  assert(!Out.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!Out.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1236, __extension__ __PRETTY_FUNCTION__));
  assert(Out.IsFixed && "non-fixed return value is not valid")(static_cast <bool> (Out.IsFixed && "non-fixed return value is not valid"
) ? void (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1237, __extension__ __PRETTY_FUNCTION__));
  if (Out.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
  if (Out.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
  if (Out.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
}

return Chain;
1247}

1249SDValue WebAssemblyTargetLowering::LowerFormalArguments(
  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
if (!callingConvSupported(CallConv))
  fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");

MachineFunction &MF = DAG.getMachineFunction();
auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();

// Set up the incoming ARGUMENTS value, which serves to represent the liveness
// of the incoming values before they're represented by virtual registers.
MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);

bool HasSwiftErrorArg = false;
bool HasSwiftSelfArg = false;
for (const ISD::InputArg &In : Ins) {
  HasSwiftSelfArg |= In.Flags.isSwiftSelf();
  HasSwiftErrorArg |= In.Flags.isSwiftError();
  if (In.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
  if (In.Flags.isNest())
    fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
  if (In.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
  if (In.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
  // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
  // registers.
  InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
                                         DAG.getTargetConstant(InVals.size(),
                                                               DL, MVT::i32))
                           : DAG.getUNDEF(In.VT));

  // Record the number and types of arguments.
  MFI->addParam(In.VT);
}

// For swiftcc, emit additional swiftself and swifterror arguments
// if there aren't. These additional arguments are also added for callee
// signature They are necessary to match callee and caller signature for
// indirect call.
auto PtrVT = getPointerTy(MF.getDataLayout());
if (CallConv == CallingConv::Swift) {
  if (!HasSwiftSelfArg) {
    MFI->addParam(PtrVT);
  }
  if (!HasSwiftErrorArg) {
    MFI->addParam(PtrVT);
  }
}
// Varargs are copied into a buffer allocated by the caller, and a pointer to
// the buffer is passed as an argument.
if (IsVarArg) {
  MVT PtrVT = getPointerTy(MF.getDataLayout());
  Register VarargVreg =
      MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
  MFI->setVarargBufferVreg(VarargVreg);
  Chain = DAG.getCopyToReg(
      Chain, DL, VarargVreg,
      DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
                  DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
  MFI->addParam(PtrVT);
}

// Record the number and types of arguments and results.
SmallVector<MVT, 4> Params;
SmallVector<MVT, 4> Results;
computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(),
                    MF.getFunction(), DAG.getTarget(), Params, Results);
for (MVT VT : Results)
  MFI->addResult(VT);
// TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
// the param logic here with ComputeSignatureVTs
assert(MFI->getParams().size() == Params.size() &&(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
 ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__))
       std::equal(MFI->getParams().begin(), MFI->getParams().end(),(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
 ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__))
                  Params.begin()))(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
 ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__));

return Chain;
1328}

1330void WebAssemblyTargetLowering::ReplaceNodeResults(
  SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
switch (N->getOpcode()) {
case ISD::SIGN_EXTEND_INREG:
  // Do not add any results, signifying that N should not be custom lowered
  // after all. This happens because simd128 turns on custom lowering for
  // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
  // illegal type.
  break;
default:
  llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1341)
      "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1341);
}
1343}

1345//===----------------------------------------------------------------------===//
1346//  Custom lowering hooks.
1347//===----------------------------------------------------------------------===//

1349SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
                                                SelectionDAG &DAG) const {
SDLoc DL(Op);
switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:'  at line 1389→
default:
  llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1354);
  return SDValue();
case ISD::FrameIndex:
  return LowerFrameIndex(Op, DAG);
case ISD::GlobalAddress:
  return LowerGlobalAddress(Op, DAG);
case ISD::GlobalTLSAddress:
  return LowerGlobalTLSAddress(Op, DAG);
case ISD::ExternalSymbol:
  return LowerExternalSymbol(Op, DAG);
case ISD::JumpTable:
  return LowerJumpTable(Op, DAG);
case ISD::BR_JT:
  return LowerBR_JT(Op, DAG);
case ISD::VASTART:
  return LowerVASTART(Op, DAG);
case ISD::BlockAddress:
case ISD::BRIND:
  fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
  return SDValue();
case ISD::RETURNADDR:
  return LowerRETURNADDR(Op, DAG);
case ISD::FRAMEADDR:
  return LowerFRAMEADDR(Op, DAG);
case ISD::CopyToReg:
  return LowerCopyToReg(Op, DAG);
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
  return LowerAccessVectorElement(Op, DAG);
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_WO_CHAIN:
case ISD::INTRINSIC_W_CHAIN:
  return LowerIntrinsic(Op, DAG);
case ISD::SIGN_EXTEND_INREG:
  return LowerSIGN_EXTEND_INREG(Op, DAG);
case ISD::BUILD_VECTOR:
  return LowerBUILD_VECTOR(Op, DAG);
2
←
Calling 'WebAssemblyTargetLowering::LowerBUILD_VECTOR'→
case ISD::VECTOR_SHUFFLE:
  return LowerVECTOR_SHUFFLE(Op, DAG);
case ISD::SETCC:
  return LowerSETCC(Op, DAG);
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
  return LowerShift(Op, DAG);
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
  return LowerFP_TO_INT_SAT(Op, DAG);
case ISD::LOAD:
  return LowerLoad(Op, DAG);
case ISD::STORE:
  return LowerStore(Op, DAG);
}
1407}

1409static bool IsWebAssemblyGlobal(SDValue Op) {
if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
  return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace());

return false;
1414}

1416static Optional<unsigned> IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG) {
const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
if (!FI)
  return None;

auto &MF = DAG.getMachineFunction();
return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex());
1423}

1425SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
const SDValue &Value = SN->getValue();
const SDValue &Base = SN->getBasePtr();
const SDValue &Offset = SN->getOffset();

if (IsWebAssemblyGlobal(Base)) {
  if (!Offset->isUndef())
    report_fatal_error("unexpected offset when storing to webassembly global",
                       false);

  SDVTList Tys = DAG.getVTList(MVT::Other);
  SDValue Ops[] = {SN->getChain(), Value, Base};
  return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
                                 SN->getMemoryVT(), SN->getMemOperand());
}

if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
  if (!Offset->isUndef())
    report_fatal_error("unexpected offset when storing to webassembly local",
                       false);

  SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
  SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
  SDValue Ops[] = {SN->getChain(), Idx, Value};
  return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
}

return Op;
1456}

1458SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
                                           SelectionDAG &DAG) const {
SDLoc DL(Op);
LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
const SDValue &Base = LN->getBasePtr();
const SDValue &Offset = LN->getOffset();

if (IsWebAssemblyGlobal(Base)) {
  if (!Offset->isUndef())
    report_fatal_error(
        "unexpected offset when loading from webassembly global", false);

  SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
  SDValue Ops[] = {LN->getChain(), Base};
  return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
                                 LN->getMemoryVT(), LN->getMemOperand());
}

if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
  if (!Offset->isUndef())
    report_fatal_error(
        "unexpected offset when loading from webassembly local", false);

  SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
  EVT LocalVT = LN->getValueType(0);
  SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
                                 {LN->getChain(), Idx});
  SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
  assert(Result->getNumValues() == 2 && "Loads must carry a chain!")(static_cast <bool> (Result->getNumValues() == 2 &&
 "Loads must carry a chain!") ? void (0) : __assert_fail ("Result->getNumValues() == 2 && \"Loads must carry a chain!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1486, __extension__ __PRETTY_FUNCTION__));
  return Result;
}

return Op;
1491}

1493SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
                                                SelectionDAG &DAG) const {
SDValue Src = Op.getOperand(2);
if (isa<FrameIndexSDNode>(Src.getNode())) {
  // CopyToReg nodes don't support FrameIndex operands. Other targets select
  // the FI to some LEA-like instruction, but since we don't have that, we
  // need to insert some kind of instruction that can take an FI operand and
  // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
  // local.copy between Op and its FI operand.
  SDValue Chain = Op.getOperand(0);
  SDLoc DL(Op);
  unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
  EVT VT = Src.getValueType();
  SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
                                                 : WebAssembly::COPY_I64,
                                  DL, VT, Src),
               0);
  return Op.getNode()->getNumValues() == 1
             ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
             : DAG.getCopyToReg(Chain, DL, Reg, Copy,
                                Op.getNumOperands() == 4 ? Op.getOperand(3)
                                                         : SDValue());
}
return SDValue();
1517}

1519SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
                                                 SelectionDAG &DAG) const {
int FI = cast<FrameIndexSDNode>(Op)->getIndex();
return DAG.getTargetFrameIndex(FI, Op.getValueType());
1523}

1525SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
                                                 SelectionDAG &DAG) const {
SDLoc DL(Op);

if (!Subtarget->getTargetTriple().isOSEmscripten()) {
  fail(DL, DAG,
       "Non-Emscripten WebAssembly hasn't implemented "
       "__builtin_return_address");
  return SDValue();
}

if (verifyReturnAddressArgumentIsConstant(Op, DAG))
  return SDValue();

unsigned Depth = Op.getConstantOperandVal(0);
MakeLibCallOptions CallOptions;
return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
                   {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
    .first;
1544}

1546SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
                                                SelectionDAG &DAG) const {
// Non-zero depths are not supported by WebAssembly currently. Use the
// legalizer's default expansion, which is to return 0 (what this function is
// documented to do).
if (Op.getConstantOperandVal(0) > 0)
  return SDValue();

DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
EVT VT = Op.getValueType();
Register FP =
    Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1559}

1561SDValue
1562WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
                                               SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *GA = cast<GlobalAddressSDNode>(Op);

MachineFunction &MF = DAG.getMachineFunction();
if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
  report_fatal_error("cannot use thread-local storage without bulk memory",
                     false);

const GlobalValue *GV = GA->getGlobal();

// Currently Emscripten does not support dynamic linking with threads.
// Therefore, if we have thread-local storage, only the local-exec model
// is possible.
// TODO: remove this and implement proper TLS models once Emscripten
// supports dynamic linking with threads.
if (GV->getThreadLocalMode() != GlobalValue::LocalExecTLSModel &&
    !Subtarget->getTargetTriple().isOSEmscripten()) {
  report_fatal_error("only -ftls-model=local-exec is supported for now on "
                     "non-Emscripten OSes: variable " +
                         GV->getName(),
                     false);
}

auto model = GV->getThreadLocalMode();

// Unsupported TLS modes
assert(model != GlobalValue::NotThreadLocal)(static_cast <bool> (model != GlobalValue::NotThreadLocal
) ? void (0) : __assert_fail ("model != GlobalValue::NotThreadLocal"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1590, __extension__ __PRETTY_FUNCTION__));
assert(model != GlobalValue::InitialExecTLSModel)(static_cast <bool> (model != GlobalValue::InitialExecTLSModel
) ? void (0) : __assert_fail ("model != GlobalValue::InitialExecTLSModel"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1591, __extension__ __PRETTY_FUNCTION__));

if (model == GlobalValue::LocalExecTLSModel ||
    model == GlobalValue::LocalDynamicTLSModel ||
    (model == GlobalValue::GeneralDynamicTLSModel &&
     getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) {
  // For DSO-local TLS variables we use offset from __tls_base

  MVT PtrVT = getPointerTy(DAG.getDataLayout());
  auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
                                     : WebAssembly::GLOBAL_GET_I32;
  const char *BaseName = MF.createExternalSymbolName("__tls_base");

  SDValue BaseAddr(
      DAG.getMachineNode(GlobalGet, DL, PtrVT,
                         DAG.getTargetExternalSymbol(BaseName, PtrVT)),
      0);

  SDValue TLSOffset = DAG.getTargetGlobalAddress(
      GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
  SDValue SymOffset =
      DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);

  return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
}

assert(model == GlobalValue::GeneralDynamicTLSModel)(static_cast <bool> (model == GlobalValue::GeneralDynamicTLSModel
) ? void (0) : __assert_fail ("model == GlobalValue::GeneralDynamicTLSModel"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1617, __extension__ __PRETTY_FUNCTION__));

EVT VT = Op.getValueType();
return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                   DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
                                              GA->getOffset(),
                                              WebAssemblyII::MO_GOT_TLS));
1624}

1626SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
                                                    SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *GA = cast<GlobalAddressSDNode>(Op);
EVT VT = Op.getValueType();
assert(GA->getTargetFlags() == 0 &&(static_cast <bool> (GA->getTargetFlags() == 0 &&
 "Unexpected target flags on generic GlobalAddressSDNode") ? void
 (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1632, __extension__ __PRETTY_FUNCTION__))
       "Unexpected target flags on generic GlobalAddressSDNode")(static_cast <bool> (GA->getTargetFlags() == 0 &&
 "Unexpected target flags on generic GlobalAddressSDNode") ? void
 (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1632, __extension__ __PRETTY_FUNCTION__));
if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace()))
  fail(DL, DAG, "Invalid address space for WebAssembly target");

unsigned OperandFlags = 0;
if (isPositionIndependent()) {
  const GlobalValue *GV = GA->getGlobal();
  if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
    MachineFunction &MF = DAG.getMachineFunction();
    MVT PtrVT = getPointerTy(MF.getDataLayout());
    const char *BaseName;
    if (GV->getValueType()->isFunctionTy()) {
      BaseName = MF.createExternalSymbolName("__table_base");
      OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
    }
    else {
      BaseName = MF.createExternalSymbolName("__memory_base");
      OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
    }
    SDValue BaseAddr =
        DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
                    DAG.getTargetExternalSymbol(BaseName, PtrVT));

    SDValue SymAddr = DAG.getNode(
        WebAssemblyISD::WrapperREL, DL, VT,
        DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
                                   OperandFlags));

    return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
  }
  OperandFlags = WebAssemblyII::MO_GOT;
}

return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                   DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
                                              GA->getOffset(), OperandFlags));
1668}

1670SDValue
1671WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
                                             SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *ES = cast<ExternalSymbolSDNode>(Op);
EVT VT = Op.getValueType();
assert(ES->getTargetFlags() == 0 &&(static_cast <bool> (ES->getTargetFlags() == 0 &&
 "Unexpected target flags on generic ExternalSymbolSDNode") ?
 void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1677, __extension__ __PRETTY_FUNCTION__))
       "Unexpected target flags on generic ExternalSymbolSDNode")(static_cast <bool> (ES->getTargetFlags() == 0 &&
 "Unexpected target flags on generic ExternalSymbolSDNode") ?
 void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1677, __extension__ __PRETTY_FUNCTION__));
return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                   DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1680}

1682SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
                                                SelectionDAG &DAG) const {
// There's no need for a Wrapper node because we always incorporate a jump
// table operand into a BR_TABLE instruction, rather than ever
// materializing it in a register.
const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
                              JT->getTargetFlags());
1690}

1692SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue Chain = Op.getOperand(0);
const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
SDValue Index = Op.getOperand(2);
assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")(static_cast <bool> (JT->getTargetFlags() == 0 &&
 "WebAssembly doesn't set target flags") ? void (0) : __assert_fail
 ("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1698, __extension__ __PRETTY_FUNCTION__));

SmallVector<SDValue, 8> Ops;
Ops.push_back(Chain);
Ops.push_back(Index);

MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;

// Add an operand for each case.
for (auto MBB : MBBs)
  Ops.push_back(DAG.getBasicBlock(MBB));

// Add the first MBB as a dummy default target for now. This will be replaced
// with the proper default target (and the preceding range check eliminated)
// if possible by WebAssemblyFixBrTableDefaults.
Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1716}

1718SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
                                              SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());

auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();

SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
                                  MFI->getVarargBufferVreg(), PtrVT);
return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
                    MachinePointerInfo(SV));
1730}

1732SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
                                                SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
unsigned IntNo;
switch (Op.getOpcode()) {
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_W_CHAIN:
  IntNo = Op.getConstantOperandVal(1);
  break;
case ISD::INTRINSIC_WO_CHAIN:
  IntNo = Op.getConstantOperandVal(0);
  break;
default:
  llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1745);
}
SDLoc DL(Op);

switch (IntNo) {
default:
  return SDValue(); // Don't custom lower most intrinsics.

case Intrinsic::wasm_lsda: {
  auto PtrVT = getPointerTy(MF.getDataLayout());
  const char *SymName = MF.createExternalSymbolName(
      "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
  if (isPositionIndependent()) {
    SDValue Node = DAG.getTargetExternalSymbol(
        SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
    const char *BaseName = MF.createExternalSymbolName("__memory_base");
    SDValue BaseAddr =
        DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
                    DAG.getTargetExternalSymbol(BaseName, PtrVT));
    SDValue SymAddr =
        DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
    return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
  }
  SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
  return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
}

case Intrinsic::wasm_shuffle: {
  // Drop in-chain and replace undefs, but otherwise pass through unchanged
  SDValue Ops[18];
  size_t OpIdx = 0;
  Ops[OpIdx++] = Op.getOperand(1);
  Ops[OpIdx++] = Op.getOperand(2);
  while (OpIdx < 18) {
    const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
    if (MaskIdx.isUndef() ||
        cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) {
      Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32);
    } else {
      Ops[OpIdx++] = MaskIdx;
    }
  }
  return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
}
}
1790}

1792SDValue
1793WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
                                                SelectionDAG &DAG) const {
SDLoc DL(Op);
// If sign extension operations are disabled, allow sext_inreg only if operand
// is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
// extension operations, but allowing sext_inreg in this context lets us have
// simple patterns to select extract_lane_s instructions. Expanding sext_inreg
// everywhere would be simpler in this file, but would necessitate large and
// brittle patterns to undo the expansion and select extract_lane_s
// instructions.
assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())(static_cast <bool> (!Subtarget->hasSignExt() &&
 Subtarget->hasSIMD128()) ? void (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1803, __extension__ __PRETTY_FUNCTION__));
if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
  return SDValue();

const SDValue &Extract = Op.getOperand(0);
MVT VecT = Extract.getOperand(0).getSimpleValueType();
if (VecT.getVectorElementType().getSizeInBits() > 32)
  return SDValue();
MVT ExtractedLaneT =
    cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
MVT ExtractedVecT =
    MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
if (ExtractedVecT == VecT)
  return Op;

// Bitcast vector to appropriate type to ensure ISel pattern coverage
const SDNode *Index = Extract.getOperand(1).getNode();
if (!isa<ConstantSDNode>(Index))
  return SDValue();
unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue();
unsigned Scale =
    ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail
 ("Scale > 1", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1825, __extension__ __PRETTY_FUNCTION__));
SDValue NewIndex =
    DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
SDValue NewExtract = DAG.getNode(
    ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
    DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
                   Op.getOperand(1));
1833}

1835static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {
SDLoc DL(Op);
if (Op.getValueType() != MVT::v2f64)
  return SDValue();

auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
                           unsigned &Index) -> bool {
  switch (Op.getOpcode()) {
  case ISD::SINT_TO_FP:
    Opcode = WebAssemblyISD::CONVERT_LOW_S;
    break;
  case ISD::UINT_TO_FP:
    Opcode = WebAssemblyISD::CONVERT_LOW_U;
    break;
  case ISD::FP_EXTEND:
    Opcode = WebAssemblyISD::PROMOTE_LOW;
    break;
  default:
    return false;
  }

  auto ExtractVector = Op.getOperand(0);
  if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return false;

  if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
    return false;

  SrcVec = ExtractVector.getOperand(0);
  Index = ExtractVector.getConstantOperandVal(1);
  return true;
};

unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
SDValue LHSSrcVec, RHSSrcVec;
if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
    !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
  return SDValue();

if (LHSOpcode != RHSOpcode)
  return SDValue();

MVT ExpectedSrcVT;
switch (LHSOpcode) {
case WebAssemblyISD::CONVERT_LOW_S:
case WebAssemblyISD::CONVERT_LOW_U:
  ExpectedSrcVT = MVT::v4i32;
  break;
case WebAssemblyISD::PROMOTE_LOW:
  ExpectedSrcVT = MVT::v4f32;
  break;
}
if (LHSSrcVec.getValueType() != ExpectedSrcVT)
  return SDValue();

auto Src = LHSSrcVec;
if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
  // Shuffle the source vector so that the converted lanes are the low lanes.
  Src = DAG.getVectorShuffle(
      ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,
      {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
}
return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);
1898}

1900SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                                   SelectionDAG &DAG) const {
if (auto ConvertLow = LowerConvertLow(Op, DAG))
3
←
Calling 'SDValue::operator bool'→
5
←
Returning from 'SDValue::operator bool'→
6
←
Taking false branch→
  return ConvertLow;

SDLoc DL(Op);
const EVT VecT = Op.getValueType();
const EVT LaneT = Op.getOperand(0).getValueType();
const size_t Lanes = Op.getNumOperands();
bool CanSwizzle = VecT == MVT::v16i8;

// BUILD_VECTORs are lowered to the instruction that initializes the highest
// possible number of lanes at once followed by a sequence of replace_lane
// instructions to individually initialize any remaining lanes.

// TODO: Tune this. For example, lanewise swizzling is very expensive, so
// swizzled lanes should be given greater weight.

// TODO: Investigate looping rather than always extracting/replacing specific
// lanes to fill gaps.

auto IsConstant = [](const SDValue &V) {
  return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
27
←
Assuming the condition is true→
28
←
Returning the value 1, which participates in a condition later→
};

// Returns the source vector and index vector pair if they exist. Checks for:
//   (extract_vector_elt
//     $src,
//     (sign_extend_inreg (extract_vector_elt $indices, $i))
//   )
auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
  auto Bail = std::make_pair(SDValue(), SDValue());
  if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return Bail;
  const SDValue &SwizzleSrc = Lane->getOperand(0);
  const SDValue &IndexExt = Lane->getOperand(1);
  if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
    return Bail;
  const SDValue &Index = IndexExt->getOperand(0);
  if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return Bail;
  const SDValue &SwizzleIndices = Index->getOperand(0);
  if (SwizzleSrc.getValueType() != MVT::v16i8 ||
      SwizzleIndices.getValueType() != MVT::v16i8 ||
      Index->getOperand(1)->getOpcode() != ISD::Constant ||
      Index->getConstantOperandVal(1) != I)
    return Bail;
  return std::make_pair(SwizzleSrc, SwizzleIndices);
};

// If the lane is extracted from another vector at a constant index, return
// that vector. The source vector must not have more lanes than the dest
// because the shufflevector indices are in terms of the destination lanes and
// would not be able to address the smaller individual source lanes.
auto GetShuffleSrc = [&](const SDValue &Lane) {
  if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return SDValue();
  if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
    return SDValue();
  if (Lane->getOperand(0).getValueType().getVectorNumElements() >
      VecT.getVectorNumElements())
    return SDValue();
  return Lane->getOperand(0);
};

using ValueEntry = std::pair<SDValue, size_t>;
SmallVector<ValueEntry, 16> SplatValueCounts;

using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
SmallVector<SwizzleEntry, 16> SwizzleCounts;

using ShuffleEntry = std::pair<SDValue, size_t>;
SmallVector<ShuffleEntry, 16> ShuffleCounts;

auto AddCount = [](auto &Counts, const auto &Val) {
  auto CountIt =
      llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
  if (CountIt == Counts.end()) {
    Counts.emplace_back(Val, 1);
  } else {
    CountIt->second++;
  }
};

auto GetMostCommon = [](auto &Counts) {
  auto CommonIt =
      std::max_element(Counts.begin(), Counts.end(),
                       [](auto A, auto B) { return A.second < B.second; });
  assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")(static_cast <bool> (CommonIt != Counts.end() &&
 "Unexpected all-undef build_vector") ? void (0) : __assert_fail
 ("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1988, __extension__ __PRETTY_FUNCTION__));
  return *CommonIt;
};

size_t NumConstantLanes = 0;

// Count eligible lanes for each type of vector creation op
for (size_t I = 0; I6.1
'I' is < 'Lanes'
1
'I' is < 'Lanes'
 < Lanes; ++I) {
7
←
Loop condition is true.  Entering loop body→
13
←
Assuming 'I' is >= 'Lanes'→
14
←
Loop condition is false. Execution continues on line 2013→
  const SDValue &Lane = Op->getOperand(I);
  if (Lane.isUndef())
8
←
Taking false branch→
    continue;

  AddCount(SplatValueCounts, Lane);

  if (IsConstant(Lane))
9
←
Taking false branch→
    NumConstantLanes++;
  if (auto ShuffleSrc = GetShuffleSrc(Lane))
10
←
Taking false branch→
    AddCount(ShuffleCounts, ShuffleSrc);
  if (CanSwizzle10.1
'CanSwizzle' is true
1
'CanSwizzle' is true
) {
11
←
Taking true branch→
    auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
    if (SwizzleSrcs.first)
12
←
Taking false branch→
      AddCount(SwizzleCounts, SwizzleSrcs);
  }
}

SDValue SplatValue;
size_t NumSplatLanes;
std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);

SDValue SwizzleSrc;
SDValue SwizzleIndices;
size_t NumSwizzleLanes = 0;
if (SwizzleCounts.size())
15
←
Assuming the condition is false→
16
←
Taking false branch→
  std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
                        NumSwizzleLanes) = GetMostCommon(SwizzleCounts);

// Shuffles can draw from up to two vectors, so find the two most common
// sources.
SDValue ShuffleSrc1, ShuffleSrc2;
size_t NumShuffleLanes = 0;
if (ShuffleCounts.size()) {
17
←
Assuming the condition is true→
18
←
Taking true branch→
  std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
  ShuffleCounts.erase(std::remove_if(ShuffleCounts.begin(),
                                     ShuffleCounts.end(),
                                     [&](const auto &Pair) {
                                       return Pair.first == ShuffleSrc1;
                                     }),
                      ShuffleCounts.end());
}
if (ShuffleCounts.size()) {
19
←
Assuming the condition is false→
20
←
Taking false branch→
  size_t AdditionalShuffleLanes;
  std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
      GetMostCommon(ShuffleCounts);
  NumShuffleLanes += AdditionalShuffleLanes;
}

// Predicate returning true if the lane is properly initialized by the
// original instruction
std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
SDValue Result;
// Prefer swizzles over shuffles over vector consts over splats
if (NumSwizzleLanes >= NumShuffleLanes &&
21
←
Assuming 'NumSwizzleLanes' is < 'NumShuffleLanes'→
    NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
  Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
                       SwizzleIndices);
  auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
  IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
    return Swizzled == GetSwizzleSrcs(I, Lane);
  };
} else if (NumShuffleLanes21.1
'NumShuffleLanes' is >= 'NumConstantLanes'
1
'NumShuffleLanes' is >= 'NumConstantLanes'
 >= NumConstantLanes &&
23
←
Taking false branch→
           NumShuffleLanes >= NumSplatLanes) {
22
←
Assuming 'NumShuffleLanes' is < 'NumSplatLanes'→
  size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
  size_t DestLaneCount = VecT.getVectorNumElements();
  size_t Scale1 = 1;
  size_t Scale2 = 1;
  SDValue Src1 = ShuffleSrc1;
  SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
  if (Src1.getValueType() != VecT) {
    size_t LaneSize =
        Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
    assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
 (0) : __assert_fail ("LaneSize > DestLaneSize", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2068, __extension__ __PRETTY_FUNCTION__));
    Scale1 = LaneSize / DestLaneSize;
    Src1 = DAG.getBitcast(VecT, Src1);
  }
  if (Src2.getValueType() != VecT) {
    size_t LaneSize =
        Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
    assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
 (0) : __assert_fail ("LaneSize > DestLaneSize", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2075, __extension__ __PRETTY_FUNCTION__));
    Scale2 = LaneSize / DestLaneSize;
    Src2 = DAG.getBitcast(VecT, Src2);
  }

  int Mask[16];
  assert(DestLaneCount <= 16)(static_cast <bool> (DestLaneCount <= 16) ? void (0)
 : __assert_fail ("DestLaneCount <= 16", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2081, __extension__ __PRETTY_FUNCTION__));
  for (size_t I = 0; I < DestLaneCount; ++I) {
    const SDValue &Lane = Op->getOperand(I);
    SDValue Src = GetShuffleSrc(Lane);
    if (Src == ShuffleSrc1) {
      Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
    } else if (Src && Src == ShuffleSrc2) {
      Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
    } else {
      Mask[I] = -1;
    }
  }
  ArrayRef<int> MaskRef(Mask, DestLaneCount);
  Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
  IsLaneConstructed = [&](size_t, const SDValue &Lane) {
    auto Src = GetShuffleSrc(Lane);
    return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
  };
} else if (NumConstantLanes >= NumSplatLanes) {
24
←
Assuming 'NumConstantLanes' is >= 'NumSplatLanes'→
25
←
Taking true branch→
  SmallVector<SDValue, 16> ConstLanes;
  for (const SDValue &Lane : Op->op_values()) {
    if (IsConstant(Lane)) {
26
←
Calling 'operator()'→
29
←
Returning from 'operator()'→
30
←
Taking true branch→
      // Values may need to be fixed so that they will sign extend to be
      // within the expected range during ISel. Check whether the value is in
      // bounds based on the lane bit width and if it is out of bounds, lop
      // off the extra bits and subtract 2^n to reflect giving the high bit
      // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it
      // cannot possibly be out of range.
      auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode());
31
←
Assuming the object is not a 'ConstantSDNode'→
      int64_t Val = Const31.1
'Const' is null
1
'Const' is null
 ? Const->getSExtValue() : 0;
32
←
'?' condition is false→
      uint64_t LaneBits = 128 / Lanes;
      assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll
 << (LaneBits - 1))) && "Unexpected out of bounds negative value"
) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2113, __extension__ __PRETTY_FUNCTION__))
33
←
The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long'
             "Unexpected out of bounds negative value")(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll
 << (LaneBits - 1))) && "Unexpected out of bounds negative value"
) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2113, __extension__ __PRETTY_FUNCTION__));
      if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
        auto NewVal = ((uint64_t)Val % (1ll << LaneBits)) - (1ll << LaneBits);
        ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT));
      } else {
        ConstLanes.push_back(Lane);
      }
    } else if (LaneT.isFloatingPoint()) {
      ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
    } else {
      ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
    }
  }
  Result = DAG.getBuildVector(VecT, DL, ConstLanes);
  IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
    return IsConstant(Lane);
  };
} else {
  // Use a splat, but possibly a load_splat
  LoadSDNode *SplattedLoad;
  if ((SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
      SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
    Result = DAG.getMemIntrinsicNode(
        WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT),
        {SplattedLoad->getChain(), SplattedLoad->getBasePtr(),
         SplattedLoad->getOffset()},
        SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand());
  } else {
    Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
  }
  IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
    return Lane == SplatValue;
  };
}

assert(Result)(static_cast <bool> (Result) ? void (0) : __assert_fail
 ("Result", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2148, __extension__ __PRETTY_FUNCTION__));
assert(IsLaneConstructed)(static_cast <bool> (IsLaneConstructed) ? void (0) : __assert_fail
 ("IsLaneConstructed", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2149, __extension__ __PRETTY_FUNCTION__));

// Add replace_lane instructions for any unhandled values
for (size_t I = 0; I < Lanes; ++I) {
  const SDValue &Lane = Op->getOperand(I);
  if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
    Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
                         DAG.getConstant(I, DL, MVT::i32));
}

return Result;
2160}

2162SDValue
2163WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
                                             SelectionDAG &DAG) const {
SDLoc DL(Op);
ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
MVT VecType = Op.getOperand(0).getSimpleValueType();
assert(VecType.is128BitVector() && "Unexpected shuffle vector type")(static_cast <bool> (VecType.is128BitVector() &&
 "Unexpected shuffle vector type") ? void (0) : __assert_fail
 ("VecType.is128BitVector() && \"Unexpected shuffle vector type\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2168, __extension__ __PRETTY_FUNCTION__));
size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;

// Space for two vector args and sixteen mask indices
SDValue Ops[18];
size_t OpIdx = 0;
Ops[OpIdx++] = Op.getOperand(0);
Ops[OpIdx++] = Op.getOperand(1);

// Expand mask indices to byte indices and materialize them as operands
for (int M : Mask) {
  for (size_t J = 0; J < LaneBytes; ++J) {
    // Lower undefs (represented by -1 in mask) to zero
    uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
    Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
  }
}

return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2187}

2189SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
// The legalizer does not know how to expand the unsupported comparison modes
// of i64x2 vectors, so we manually unroll them here.
assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64)(static_cast <bool> (Op->getOperand(0)->getSimpleValueType
(0) == MVT::v2i64) ? void (0) : __assert_fail ("Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2194, __extension__ __PRETTY_FUNCTION__));
SmallVector<SDValue, 2> LHS, RHS;
DAG.ExtractVectorElements(Op->getOperand(0), LHS);
DAG.ExtractVectorElements(Op->getOperand(1), RHS);
const SDValue &CC = Op->getOperand(2);
auto MakeLane = [&](unsigned I) {
  return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
                     DAG.getConstant(uint64_t(-1), DL, MVT::i64),
                     DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
};
return DAG.getBuildVector(Op->getValueType(0), DL,
                          {MakeLane(0), MakeLane(1)});
2206}

2208SDValue
2209WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
                                                  SelectionDAG &DAG) const {
// Allow constant lane indices, expand variable lane indices
SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
  return Op;
else
  // Perform default expansion
  return SDValue();
2218}

2220static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
EVT LaneT = Op.getSimpleValueType().getVectorElementType();
// 32-bit and 64-bit unrolled shifts will have proper semantics
if (LaneT.bitsGE(MVT::i32))
  return DAG.UnrollVectorOp(Op.getNode());
// Otherwise mask the shift value to get proper semantics from 32-bit shift
SDLoc DL(Op);
size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
unsigned ShiftOpcode = Op.getOpcode();
SmallVector<SDValue, 16> ShiftedElements;
DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
SmallVector<SDValue, 16> ShiftElements;
DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
SmallVector<SDValue, 16> UnrolledOps;
for (size_t i = 0; i < NumLanes; ++i) {
  SDValue MaskedShiftValue =
      DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
  SDValue ShiftedValue = ShiftedElements[i];
  if (ShiftOpcode == ISD::SRA)
    ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
                               ShiftedValue, DAG.getValueType(LaneT));
  UnrolledOps.push_back(
      DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
}
return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2246}

2248SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);

// Only manually lower vector shifts
assert(Op.getSimpleValueType().isVector())(static_cast <bool> (Op.getSimpleValueType().isVector()
) ? void (0) : __assert_fail ("Op.getSimpleValueType().isVector()"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2253, __extension__ __PRETTY_FUNCTION__));

auto ShiftVal = DAG.getSplatValue(Op.getOperand(1));
if (!ShiftVal)
  return unrollVectorShift(Op, DAG);

// Use anyext because none of the high bits can affect the shift
ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);

unsigned Opcode;
switch (Op.getOpcode()) {
case ISD::SHL:
  Opcode = WebAssemblyISD::VEC_SHL;
  break;
case ISD::SRA:
  Opcode = WebAssemblyISD::VEC_SHR_S;
  break;
case ISD::SRL:
  Opcode = WebAssemblyISD::VEC_SHR_U;
  break;
default:
  llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2274);
}

return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2278}

2280SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
                                                    SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT ResT = Op.getValueType();
EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

if ((ResT == MVT::i32 || ResT == MVT::i64) &&
    (SatVT == MVT::i32 || SatVT == MVT::i64))
  return Op;

if (ResT == MVT::v4i32 && SatVT == MVT::i32)
  return Op;

return SDValue();
2294}

2296//===----------------------------------------------------------------------===//
2297//   Custom DAG combine hooks
2298//===----------------------------------------------------------------------===//
2299static SDValue
2300performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;
auto Shuffle = cast<ShuffleVectorSDNode>(N);

// Hoist vector bitcasts that don't change the number of lanes out of unary
// shuffles, where they are less likely to get in the way of other combines.
// (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
//  (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
SDValue Bitcast = N->getOperand(0);
if (Bitcast.getOpcode() != ISD::BITCAST)
  return SDValue();
if (!N->getOperand(1).isUndef())
  return SDValue();
SDValue CastOp = Bitcast.getOperand(0);
MVT SrcType = CastOp.getSimpleValueType();
MVT DstType = Bitcast.getSimpleValueType();
if (!SrcType.is128BitVector() ||
    SrcType.getVectorNumElements() != DstType.getVectorNumElements())
  return SDValue();
SDValue NewShuffle = DAG.getVectorShuffle(
    SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
return DAG.getBitcast(DstType, NewShuffle);
2322}

2324static SDValue
2325performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;
assert(N->getOpcode() == ISD::SIGN_EXTEND ||(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
 || N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
 ("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__))
       N->getOpcode() == ISD::ZERO_EXTEND)(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
 || N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
 ("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__));

// Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
// possible before the extract_subvector can be expanded.
auto Extract = N->getOperand(0);
if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
  return SDValue();
auto Source = Extract.getOperand(0);
auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
if (IndexNode == nullptr)
  return SDValue();
auto Index = IndexNode->getZExtValue();

// Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
// extracted subvector is the low or high half of its source.
EVT ResVT = N->getValueType(0);
if (ResVT == MVT::v8i16) {
  if (Extract.getValueType() != MVT::v8i8 ||
      Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
    return SDValue();
} else if (ResVT == MVT::v4i32) {
  if (Extract.getValueType() != MVT::v4i16 ||
      Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
    return SDValue();
} else if (ResVT == MVT::v2i64) {
  if (Extract.getValueType() != MVT::v2i32 ||
      Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
    return SDValue();
} else {
  return SDValue();
}

bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
bool IsLow = Index == 0;

unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
                              : WebAssemblyISD::EXTEND_HIGH_S)
                     : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
                              : WebAssemblyISD::EXTEND_HIGH_U);

return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2369}

2371static SDValue
2372performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;

auto GetWasmConversionOp = [](unsigned Op) {
  switch (Op) {
  case ISD::FP_TO_SINT_SAT:
    return WebAssemblyISD::TRUNC_SAT_ZERO_S;
  case ISD::FP_TO_UINT_SAT:
    return WebAssemblyISD::TRUNC_SAT_ZERO_U;
  case ISD::FP_ROUND:
    return WebAssemblyISD::DEMOTE_ZERO;
  }
  llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2384);
};

auto IsZeroSplat = [](SDValue SplatVal) {
  auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
  APInt SplatValue, SplatUndef;
  unsigned SplatBitSize;
  bool HasAnyUndefs;
  return Splat &&
         Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
                                HasAnyUndefs) &&
         SplatValue == 0;
};

if (N->getOpcode() == ISD::CONCAT_VECTORS) {
  // Combine this:
  //
  //   (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
  //
  // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
  //
  // Or this:
  //
  //   (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
  //
  // into (f32x4.demote_zero_f64x2 $x).
  EVT ResVT;
  EVT ExpectedConversionType;
  auto Conversion = N->getOperand(0);
  auto ConversionOp = Conversion.getOpcode();
  switch (ConversionOp) {
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
    ResVT = MVT::v4i32;
    ExpectedConversionType = MVT::v2i32;
    break;
  case ISD::FP_ROUND:
    ResVT = MVT::v4f32;
    ExpectedConversionType = MVT::v2f32;
    break;
  default:
    return SDValue();
  }

  if (N->getValueType(0) != ResVT)
    return SDValue();

  if (Conversion.getValueType() != ExpectedConversionType)
    return SDValue();

  auto Source = Conversion.getOperand(0);
  if (Source.getValueType() != MVT::v2f64)
    return SDValue();

  if (!IsZeroSplat(N->getOperand(1)) ||
      N->getOperand(1).getValueType() != ExpectedConversionType)
    return SDValue();

  unsigned Op = GetWasmConversionOp(ConversionOp);
  return DAG.getNode(Op, SDLoc(N), ResVT, Source);
}

// Combine this:
//
//   (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
//
// into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
//
// Or this:
//
//   (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
//
// into (f32x4.demote_zero_f64x2 $x).
EVT ResVT;
auto ConversionOp = N->getOpcode();
switch (ConversionOp) {
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
  ResVT = MVT::v4i32;
  break;
case ISD::FP_ROUND:
  ResVT = MVT::v4f32;
  break;
default:
  llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2468);
}

if (N->getValueType(0) != ResVT)
  return SDValue();

auto Concat = N->getOperand(0);
if (Concat.getValueType() != MVT::v4f64)
  return SDValue();

auto Source = Concat.getOperand(0);
if (Source.getValueType() != MVT::v2f64)
  return SDValue();

if (!IsZeroSplat(Concat.getOperand(1)) ||
    Concat.getOperand(1).getValueType() != MVT::v2f64)
  return SDValue();

unsigned Op = GetWasmConversionOp(ConversionOp);
return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2488}

2490SDValue
2491WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
                                           DAGCombinerInfo &DCI) const {
switch (N->getOpcode()) {
default:
  return SDValue();
case ISD::VECTOR_SHUFFLE:
  return performVECTOR_SHUFFLECombine(N, DCI);
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
  return performVectorExtendCombine(N, DCI);
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
case ISD::FP_ROUND:
case ISD::CONCAT_VECTORS:
  return performVectorTruncZeroCombine(N, DCI);
}
2507}

←

/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>

57namespace llvm {

59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;

72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG.  Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
82};

84namespace ISD {

/// Node predicates

88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);

109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);

113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);

117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);

125} // end namespace ISD

127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation.  Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node.  This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
unsigned ResNo = 0;     // Which return value of the node we are using.

144public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
4
←
Returning zero, which participates in a condition later→
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
230};

232template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
253};

255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
}
263};
264template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
270};

272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

287public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

327private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
356};

358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
366};

368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

391public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
451};

453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
/// The operation that this node performs.
int16_t NodeType;

460protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with NodeType.

465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif

475BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class VPLoadStoreSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;
  friend class VPGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   VPGatherScatterSDNode => enum ISD::MemIndexType
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class VPLoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;
  friend class VPGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class VPStoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;
  friend class VPScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
565END_TWO_BYTE_PACK()
566#undef BEGIN_TWO_BYTE_PACK
567#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

579private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

616public:
/// Unique and persistent id per SDNode in the DAG.
/// Used for debug printing.
uint16_t PersistentId;

//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned short)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
672#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
674#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 687, __extension__ __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using iterator_category = std::forward_iterator_tag;
  using value_type = SDUse;
  using difference_type = std::ptrdiff_t;
  using pointer = value_type *;
  using reference = value_type &;

  use_iterator() = default;
  use_iterator(const use_iterator &I) : Op(I.Op) {}

  bool operator==(const use_iterator &x) const {
    return Op == x.Op;
  }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 756, __extension__ __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 767, __extension__ __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 777, __extension__ __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 905, __extension__ __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 967, __extension__ __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1054protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
 "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1068, __extension__ __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1075};

1077/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1078/// into SDNode creation functions.
1079/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1080/// from the original Instruction, and IROrder is the ordinal position of
1081/// the instruction.
1082/// When an SDNode is created after the DAG is being built, both DebugLoc and
1083/// the IROrder are propagated from the original SDNode.
1084/// So SDLoc class provides two constructors besides the default one, one to
1085/// be used by the DAGBuilder, the other to be used by others.
1086class SDLoc {
1087private:
DebugLoc DL;
int IROrder = 0;

1091public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1096, __extension__ __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1103};

1105// Define inline functions from the SDValue class.

1107inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1114, __extension__ __PRETTY_FUNCTION__));
1115}

1117inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
1119}

1121inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
1123}

1125inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1127}

1129inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1131}

1133inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1135}

1137inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1139}

1141inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1143}

1145inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1147}

1149inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1151}

1153inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1155}

1157inline bool SDValue::isUndef() const {
return Node->isUndef();
1159}

1161inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1163}

1165inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
1167}

1169inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1171}

1173inline void SDValue::dump() const {
return Node->dump();
1175}

1177inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1179}

1181inline void SDValue::dumpr() const {
return Node->dumpr();
1183}

1185inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1187}

1189// Define inline functions from the SDUse class.

1191inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode()) V.getNode()->addUse(*this);
1195}

1197inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V.getNode()->addUse(*this);
1200}

1202inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1206}

1208/// This class is used to form a handle around another node that
1209/// is persistent and is updated across invocations of replaceAllUsesWith on its
1210/// operand.  This node should be directly created by end-users and not added to
1211/// the AllNodes list.
1212class HandleSDNode : public SDNode {
SDUse Op;

1215public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1234};

1236class AddrSpaceCastSDNode : public SDNode {
1237private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1241public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1251};

1253/// This is an abstract virtual class for memory operations.
1254class MemSDNode : public SDNode {
1255private:
// VT of in-memory value.
EVT MemoryVT;

1259protected:
/// Memory reference information.
MachineMemOperand *MMO;

1263public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }
// FIXME: Remove once transition to getAlign is over.
unsigned getAlignment() const { return MMO->getAlign().value(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getSuccessOrdering() const {
  return MMO->getSuccessOrdering();
}

/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation.  (For operations
/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::VP_STORE:
  case ISD::MSTORE:
  case ISD::VP_SCATTER:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  switch (N->getOpcode()) {
  case ISD::LOAD:
  case ISD::STORE:
  case ISD::PREFETCH:
  case ISD::ATOMIC_CMP_SWAP:
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  case ISD::ATOMIC_SWAP:
  case ISD::ATOMIC_LOAD_ADD:
  case ISD::ATOMIC_LOAD_SUB:
  case ISD::ATOMIC_LOAD_AND:
  case ISD::ATOMIC_LOAD_CLR:
  case ISD::ATOMIC_LOAD_OR:
  case ISD::ATOMIC_LOAD_XOR:
  case ISD::ATOMIC_LOAD_NAND:
  case ISD::ATOMIC_LOAD_MIN:
  case ISD::ATOMIC_LOAD_MAX:
  case ISD::ATOMIC_LOAD_UMIN:
  case ISD::ATOMIC_LOAD_UMAX:
  case ISD::ATOMIC_LOAD_FADD:
  case ISD::ATOMIC_LOAD_FSUB:
  case ISD::ATOMIC_LOAD:
  case ISD::ATOMIC_STORE:
  case ISD::MLOAD:
  case ISD::MSTORE:
  case ISD::MGATHER:
  case ISD::MSCATTER:
  case ISD::VP_LOAD:
  case ISD::VP_STORE:
  case ISD::VP_GATHER:
  case ISD::VP_SCATTER:
    return true;
  default:
    return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
  }
}
1415};

1417/// This is an SDNode representing atomic operations.
1418class AtomicSDNode : public MemSDNode {
1419public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1441, __extension__ __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1466};

1468/// This SDNode is used for target intrinsics that touch
1469/// memory and need an associated MachineMemOperand. Its opcode may be
1470/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1471/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1472class MemIntrinsicSDNode : public MemSDNode {
1473public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1488};

1490/// This SDNode is used to implement the code generator
1491/// support for the llvm IR shufflevector instruction.  It combines elements
1492/// from two input vectors into a new input vector, with the selection and
1493/// ordering of elements determined by an array of integers, referred to as
1494/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1495/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1496/// An index of -1 is treated as undef, such that the code generator may put
1497/// any value in the corresponding element of the result.
1498class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1503protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1509public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return makeArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
 ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1516, __extension__ __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1523, __extension__ __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1554};

1556class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1568public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isZero() const { return Value->isZero(); }
// NOTE: This is soft-deprecated.  Please use `isZero()` instead.
bool isNullValue() const { return isZero(); }
bool isAllOnes() const { return Value->isMinusOne(); }
// NOTE: This is soft-deprecated.  Please use `isAllOnes()` instead.
bool isAllOnesValue() const { return isAllOnes(); }
bool isMaxSignedValue() const { return Value->isMaxValue(true); }
bool isMinSignedValue() const { return Value->isMinValue(true); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1595};

1597uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1599}

1601const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1603}

1605class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1615public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1650};

1652/// Returns true if \p V is a constant integer zero.
1653bool isNullConstant(SDValue V);

1655/// Returns true if \p V is an FP constant with a value of positive zero.
1656bool isNullFPConstant(SDValue V);

1658/// Returns true if \p V is an integer constant with all bits set.
1659bool isAllOnesConstant(SDValue V);

1661/// Returns true if \p V is a constant integer one.
1662bool isOneConstant(SDValue V);

1664/// Return the non-bitcasted source operand of \p V if it exists.
1665/// If \p V is not a bitcasted value, it is returned as-is.
1666SDValue peekThroughBitcasts(SDValue V);

1668/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1669/// If \p V is not a bitcasted one-use value, it is returned as-is.
1670SDValue peekThroughOneUseBitcasts(SDValue V);

1672/// Return the non-extracted vector source operand of \p V if it exists.
1673/// If \p V is not an extracted subvector, it is returned as-is.
1674SDValue peekThroughExtractSubvectors(SDValue V);

1676/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1677/// constant is canonicalized to be operand 1.
1678bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1680/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1681ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1684/// Returns the SDNode if it is a demanded constant splat BuildVector or
1685/// constant int.
1686ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1690/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1691ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1693/// Returns the SDNode if it is a demanded constant splat BuildVector or
1694/// constant float.
1695ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1698/// Return true if the value is a constant 0 integer or a splatted vector of
1699/// a constant 0 integer (with no undefs by default).
1700/// Build vector implicit truncation is not an issue for null values.
1701bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1703/// Return true if the value is a constant 1 integer or a splatted vector of a
1704/// constant 1 integer (with no undefs).
1705/// Does not permit build vector implicit truncation.
1706bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1708/// Return true if the value is a constant -1 integer or a splatted vector of a
1709/// constant -1 integer (with no undefs).
1710/// Does not permit build vector implicit truncation.
1711bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1713/// Return true if \p V is either a integer or FP constant.
1714inline bool isIntOrFPConstant(SDValue V) {
return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1716}

1718class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1729public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1742};

1744class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1754public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1761};

1763/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1764/// the offet and size that are started/ended in the underlying FrameIndex.
1765class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1773public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1780, __extension__ __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1784, __extension__ __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1793};

1795/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1796/// the index of the basic block being probed. A pseudo probe serves as a place
1797/// holder and will be removed at the end of compilation. It does not have any
1798/// operand because we do not want the instruction selection to deal with any.
1799class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1810public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1819};

1821class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1832public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1840};

1842class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1858, __extension__ __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1867, __extension__ __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1872public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1878, __extension__ __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1883, __extension__ __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1902};

1904/// Completely target-dependent object reference.
1905class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1912public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1924};

1926class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1938public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1944};

1946/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1947class BuildVectorSDNode : public SDNode {
1948public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

bool isConstant() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2057};

2059/// An SDNode that holds an arbitrary LLVM IR Value. This is
2060/// used when the SelectionDAG needs to make a simple reference to something
2061/// in the LLVM IR representation.
2062///
2063class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2072public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2079};

2081class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2090public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2096};

2098class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2106public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2112};

2114class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2124public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2130};

2132class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2144public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2153};

2155class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
 "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2162, __extension__ __PRETTY_FUNCTION__));
}

2165public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2172};

2174class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2185public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2193};

2195class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2203public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2209};

2211class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2220public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2226};

2228/// This class is used to represent EVT's, which are used
2229/// to parameterize some operations.
2230class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2239public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2245};

2247/// Base class for LoadSDNode and StoreSDNode
2248class LSBaseSDNode : public MemSDNode {
2249public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2255, __extension__ __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2278};

2280/// This class is used to represent ISD::LOAD nodes.
2281class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2289, __extension__ __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2290, __extension__ __PRETTY_FUNCTION__));
}

2293public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2306};

2308/// This class is used to represent ISD::STORE nodes.
2309class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2317, __extension__ __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2318, __extension__ __PRETTY_FUNCTION__));
}

2321public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2337};

2339/// This base class is used to represent VP_LOAD and VP_STORE nodes
2340class VPLoadStoreSDNode : public MemSDNode {
2341public:
friend class SelectionDAG;

VPLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
                  SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
                  MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2349, __extension__ __PRETTY_FUNCTION__));
}

// VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
// VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
// Mask is a vector of i1 elements;
// the type of EVL is TLI.getVPExplicitVectorLengthTy().
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 2 : 3);
}
const SDValue &getBasePtr() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 1 : 2);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 3 : 4);
}
const SDValue &getVectorLength() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 4 : 5);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
}
2384};

2386/// This class is used to represent a VP_LOAD node
2387class VPLoadSDNode : public VPLoadStoreSDNode {
2388public:
friend class SelectionDAG;

VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
             ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
             EVT MemVT, MachineMemOperand *MMO)
    : VPLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = isExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getVectorLength() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2412};

2414/// This class is used to represent a VP_STORE node
2415class VPStoreSDNode : public VPLoadStoreSDNode {
2416public:
friend class SelectionDAG;

VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
              ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
              EVT MemVT, MachineMemOperand *MMO)
    : VPLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_STORE;
}
2447};

2449/// This base class is used to represent MLOAD and MSTORE nodes
2450class MaskedLoadStoreSDNode : public MemSDNode {
2451public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2460, __extension__ __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2489};

2491/// This class is used to represent an MLOAD node
2492class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2493public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2518};

2520/// This class is used to represent an MSTORE node
2521class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2522public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2552};

2554/// This is a base class used to represent
2555/// VP_GATHER and VP_SCATTER nodes
2556///
2557class VPGatherScatterSDNode : public MemSDNode {
2558public:
friend class SelectionDAG;

VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                      MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2566, __extension__ __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// VPGatherSDNode  (Chain, base, index, scale, mask, vlen)
// VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
}
const SDValue &getIndex() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
}
const SDValue &getScale() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
}
const SDValue &getMask() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
}
const SDValue &getVectorLength() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER ||
         N->getOpcode() == ISD::VP_SCATTER;
}
2606};

2608/// This class is used to represent an VP_GATHER node
2609///
2610class VPGatherSDNode : public VPGatherScatterSDNode {
2611public:
friend class SelectionDAG;

VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
               MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER;
}
2622};

2624/// This class is used to represent an VP_SCATTER node
2625///
2626class VPScatterSDNode : public VPGatherScatterSDNode {
2627public:
friend class SelectionDAG;

VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_SCATTER;
}
2640};

2642/// This is a base class used to represent
2643/// MGATHER and MSCATTER nodes
2644///
2645class MaskedGatherScatterSDNode : public MemSDNode {
2646public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2654, __extension__ __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
void setIndexType(ISD::MemIndexType IndexType) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2686};

2688/// This class is used to represent an MGATHER node
2689///
2690class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2691public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2711};

2713/// This class is used to represent an MSCATTER node
2714///
2715class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2716public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2737};

2739/// An SDNode that represents everything that will be needed
2740/// to construct a MachineInstr. These nodes are created during the
2741/// instruction selection proper phase.
2742///
2743/// Note that the only supported way to set the `memoperands` is by calling the
2744/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2745/// inside the DAG rather than in the node.
2746class MachineSDNode : public SDNode {
2747private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2775public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2801};

2803/// An SDNode that records if a register contains a value that is guaranteed to
2804/// be aligned accordingly.
2805class AssertAlignSDNode : public SDNode {
Align Alignment;

2808public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2817};

2819class SDNodeIterator {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2825public:
using iterator_category = std::forward_iterator_tag;
using value_type = SDNode;
using difference_type = std::ptrdiff_t;
using pointer = value_type *;
using reference = value_type &;

bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2851, __extension__ __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211017100700+d245f2e8597b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2851, __extension__ __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
2862};

2864template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
2877};

2879/// A representation of the largest SDNode, for use in sizeof().
2880///
2881/// This needs to be a union because the largest node differs on 32 bit systems
2882/// with 4 and 8 byte pointer alignment, respectively.
2883using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

2888/// The SDNode class with the greatest alignment requirement.
2889using MostAlignedSDNode = GlobalAddressSDNode;

2891namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

2969} // end namespace ISD

2971} // end namespace llvm

2973#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H