/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Bug Summary

File:	llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1114, column 10 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name WebAssemblyISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/lib/Target/WebAssembly -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/build-llvm/lib/Target/WebAssembly -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-07-26-235520-9401-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

→

1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//

14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "Utils/WebAssemblyTypeUtilities.h"
17#include "Utils/WebAssemblyUtilities.h"
18#include "WebAssemblyMachineFunctionInfo.h"
19#include "WebAssemblySubtarget.h"
20#include "WebAssemblyTargetMachine.h"
21#include "llvm/CodeGen/CallingConvLower.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineModuleInfo.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/CodeGen/WasmEHFuncInfo.h"
29#include "llvm/IR/DiagnosticInfo.h"
30#include "llvm/IR/DiagnosticPrinter.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/Intrinsics.h"
33#include "llvm/IR/IntrinsicsWebAssembly.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Target/TargetOptions.h"
39using namespace llvm;

41#define DEBUG_TYPE"wasm-lower" "wasm-lower"

43WebAssemblyTargetLowering::WebAssemblyTargetLowering(
  const TargetMachine &TM, const WebAssemblySubtarget &STI)
  : TargetLowering(TM), Subtarget(&STI) {
auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;

// Booleans always contain 0 or 1.
setBooleanContents(ZeroOrOneBooleanContent);
// Except in SIMD vectors
setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
// We don't know the microarchitecture here, so just reduce register pressure.
setSchedulingPreference(Sched::RegPressure);
// Tell ISel that we have a stack pointer.
setStackPointerRegisterToSaveRestore(
    Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
// Set up the register classes.
addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
if (Subtarget->hasSIMD128()) {
  addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
  addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
}
if (Subtarget->hasReferenceTypes()) {
  addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
  addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
}
// Compute derived properties from the register classes.
computeRegisterProperties(Subtarget->getRegisterInfo());

// Transform loads and stores to pointers in address space 1 to loads and
// stores to WebAssembly global variables, outside linear memory.
for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
  setOperationAction(ISD::LOAD, T, Custom);
  setOperationAction(ISD::STORE, T, Custom);
}
if (Subtarget->hasSIMD128()) {
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64}) {
    setOperationAction(ISD::LOAD, T, Custom);
    setOperationAction(ISD::STORE, T, Custom);
  }
}
if (Subtarget->hasReferenceTypes()) {
  for (auto T : {MVT::externref, MVT::funcref}) {
    setOperationAction(ISD::LOAD, T, Custom);
    setOperationAction(ISD::STORE, T, Custom);
  }
}

setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom);
setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
setOperationAction(ISD::JumpTable, MVTPtr, Custom);
setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
setOperationAction(ISD::BRIND, MVT::Other, Custom);

// Take the default expansion for va_arg, va_copy, and va_end. There is no
// default action for va_start, so we do that custom.
setOperationAction(ISD::VASTART, MVT::Other, Custom);
setOperationAction(ISD::VAARG, MVT::Other, Expand);
setOperationAction(ISD::VACOPY, MVT::Other, Expand);
setOperationAction(ISD::VAEND, MVT::Other, Expand);

for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
  // Don't expand the floating-point types to constant pools.
  setOperationAction(ISD::ConstantFP, T, Legal);
  // Expand floating-point comparisons.
  for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
                  ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
    setCondCodeAction(CC, T, Expand);
  // Expand floating-point library function operators.
  for (auto Op :
       {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
    setOperationAction(Op, T, Expand);
  // Note supported floating-point library function operators that otherwise
  // default to expand.
  for (auto Op :
       {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
    setOperationAction(Op, T, Legal);
  // Support minimum and maximum, which otherwise default to expand.
  setOperationAction(ISD::FMINIMUM, T, Legal);
  setOperationAction(ISD::FMAXIMUM, T, Legal);
  // WebAssembly currently has no builtin f16 support.
  setOperationAction(ISD::FP16_TO_FP, T, Expand);
  setOperationAction(ISD::FP_TO_FP16, T, Expand);
  setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
  setTruncStoreAction(T, MVT::f16, Expand);
}

// Expand unavailable integer operations.
for (auto Op :
     {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
      ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
      ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
  for (auto T : {MVT::i32, MVT::i64})
    setOperationAction(Op, T, Expand);
  if (Subtarget->hasSIMD128())
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(Op, T, Expand);
}

if (Subtarget->hasNontrappingFPToInt())
  for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
    for (auto T : {MVT::i32, MVT::i64})
      setOperationAction(Op, T, Custom);

// SIMD-specific configuration
if (Subtarget->hasSIMD128()) {
  // Hoist bitcasts out of shuffles
  setTargetDAGCombine(ISD::VECTOR_SHUFFLE);

  // Combine extends of extract_subvectors into widening ops
  setTargetDAGCombine(ISD::SIGN_EXTEND);
  setTargetDAGCombine(ISD::ZERO_EXTEND);

  // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
  // conversions ops
  setTargetDAGCombine(ISD::SINT_TO_FP);
  setTargetDAGCombine(ISD::UINT_TO_FP);
  setTargetDAGCombine(ISD::FP_EXTEND);
  setTargetDAGCombine(ISD::EXTRACT_SUBVECTOR);

  // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
  // into conversion ops
  setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
  setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
  setTargetDAGCombine(ISD::FP_ROUND);
  setTargetDAGCombine(ISD::CONCAT_VECTORS);

  // Support saturating add for i8x16 and i16x8
  for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
    for (auto T : {MVT::v16i8, MVT::v8i16})
      setOperationAction(Op, T, Legal);

  // Support integer abs
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
    setOperationAction(ISD::ABS, T, Legal);

  // Custom lower BUILD_VECTORs to minimize number of replace_lanes
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64})
    setOperationAction(ISD::BUILD_VECTOR, T, Custom);

  // We have custom shuffle lowering to expose the shuffle mask
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64})
    setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);

  // Custom lowering since wasm shifts must have a scalar shift amount
  for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(Op, T, Custom);

  // Custom lower lane accesses to expand out variable indices
  for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                   MVT::v2f64})
      setOperationAction(Op, T, Custom);

  // There is no i8x16.mul instruction
  setOperationAction(ISD::MUL, MVT::v16i8, Expand);

  // There is no vector conditional select instruction
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
                 MVT::v2f64})
    setOperationAction(ISD::SELECT_CC, T, Expand);

  // Expand integer operations supported for scalars but not SIMD
  for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV,
                  ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
      setOperationAction(Op, T, Expand);

  // But we do have integer min and max operations
  for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
      setOperationAction(Op, T, Legal);

  // And we have popcnt for i8x16
  setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);

  // Expand float operations supported for scalars but not SIMD
  for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
                  ISD::FEXP, ISD::FEXP2, ISD::FRINT})
    for (auto T : {MVT::v4f32, MVT::v2f64})
      setOperationAction(Op, T, Expand);

  // Unsigned comparison operations are unavailable for i64x2 vectors.
  for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
    setCondCodeAction(CC, MVT::v2i64, Custom);

  // 64x2 conversions are not in the spec
  for (auto Op :
       {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
    for (auto T : {MVT::v2i64, MVT::v2f64})
      setOperationAction(Op, T, Expand);

  // But saturating fp_to_int converstions are
  for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
    setOperationAction(Op, MVT::v4i32, Custom);
}

// As a special case, these operators use the type to mean the type to
// sign-extend from.
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
if (!Subtarget->hasSignExt()) {
  // Sign extends are legal only when extending a vector extract
  auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
  for (auto T : {MVT::i8, MVT::i16, MVT::i32})
    setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
}
for (auto T : MVT::integer_fixedlen_vector_valuetypes())
  setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);

// Dynamic stack allocation: use the default expansion.
setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);

setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
setOperationAction(ISD::FrameIndex, MVT::i64, Custom);
setOperationAction(ISD::CopyToReg, MVT::Other, Custom);

// Expand these forms; we pattern-match the forms that we can handle in isel.
for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
  for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
    setOperationAction(Op, T, Expand);

// We have custom switch handling.
setOperationAction(ISD::BR_JT, MVT::Other, Custom);

// WebAssembly doesn't have:
//  - Floating-point extending loads.
//  - Floating-point truncating stores.
//  - i1 extending loads.
//  - truncating SIMD stores and most extending loads
setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
setTruncStoreAction(MVT::f64, MVT::f32, Expand);
for (auto T : MVT::integer_valuetypes())
  for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
    setLoadExtAction(Ext, T, MVT::i1, Promote);
if (Subtarget->hasSIMD128()) {
  for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
                 MVT::v2f64}) {
    for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
      if (MVT(T) != MemT) {
        setTruncStoreAction(T, MemT, Expand);
        for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
          setLoadExtAction(Ext, T, MemT, Expand);
      }
    }
  }
  // But some vector extending loads are legal
  for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
    setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
    setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
    setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
  }
  // And some truncating stores are legal as well
  setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
}

// Don't do anything clever with build_pairs
setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);

// Trap lowers to wasm unreachable
setOperationAction(ISD::TRAP, MVT::Other, Legal);
setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);

// Exception handling intrinsics
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);

setMaxAtomicSizeInBitsSupported(64);

// Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
// consistent with the f64 and f128 names.
setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");

// Define the emscripten name for return address helper.
// TODO: when implementing other Wasm backends, make this generic or only do
// this on emscripten depending on what they end up doing.
setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");

// Always convert switches to br_tables unless there is only one case, which
// is equivalent to a simple branch. This reduces code size for wasm, and we
// defer possible jump table optimizations to the VM.
setMinimumJumpTableEntries(2);
339}

341TargetLowering::AtomicExpansionKind
342WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
// We have wasm instructions for these
switch (AI->getOperation()) {
case AtomicRMWInst::Add:
case AtomicRMWInst::Sub:
case AtomicRMWInst::And:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor:
case AtomicRMWInst::Xchg:
  return AtomicExpansionKind::None;
default:
  break;
}
return AtomicExpansionKind::CmpXChg;
356}

358bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
// Implementation copied from X86TargetLowering.
unsigned Opc = VecOp.getOpcode();

// Assume target opcodes can't be scalarized.
// TODO - do we have any exceptions?
if (Opc >= ISD::BUILTIN_OP_END)
  return false;

// If the vector op is not supported, try to convert to scalar.
EVT VecVT = VecOp.getValueType();
if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
  return true;

// If the vector op is supported, but the scalar op is not, the transform may
// not be worthwhile.
EVT ScalarVT = VecVT.getScalarType();
return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
376}

378FastISel *WebAssemblyTargetLowering::createFastISel(
  FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
return WebAssembly::createFastISel(FuncInfo, LibInfo);
381}

383MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
                                                    EVT VT) const {
unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
if (BitWidth > 1 && BitWidth < 8)
  BitWidth = 8;

if (BitWidth > 64) {
  // The shift will be lowered to a libcall, and compiler-rt libcalls expect
  // the count to be an i32.
  BitWidth = 32;
  assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 394, __extension__ __PRETTY_FUNCTION__))
         "32-bit shift counts ought to be enough for anyone")(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 394, __extension__ __PRETTY_FUNCTION__));
}

MVT Result = MVT::getIntegerVT(BitWidth);
assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
 && "Unable to represent scalar shift amount type") ?
 void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 399, __extension__ __PRETTY_FUNCTION__))
       "Unable to represent scalar shift amount type")(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
 && "Unable to represent scalar shift amount type") ?
 void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 399, __extension__ __PRETTY_FUNCTION__));
return Result;
401}

403// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
404// undefined result on invalid/overflow, to the WebAssembly opcode, which
405// traps on invalid/overflow.
406static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
                                     MachineBasicBlock *BB,
                                     const TargetInstrInfo &TII,
                                     bool IsUnsigned, bool Int64,
                                     bool Float64, unsigned LoweredOpcode) {
MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();

Register OutReg = MI.getOperand(0).getReg();
Register InReg = MI.getOperand(1).getReg();

unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
unsigned Eqz = WebAssembly::EQZ_I32;
unsigned And = WebAssembly::AND_I32;
int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1);
int64_t Substitute = IsUnsigned ? 0 : Limit;
double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
auto &Context = BB->getParent()->getFunction().getContext();
Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);

const BasicBlock *LLVMBB = BB->getBasicBlock();
MachineFunction *F = BB->getParent();
MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);

MachineFunction::iterator It = ++BB->getIterator();
F->insert(It, FalseMBB);
F->insert(It, TrueMBB);
F->insert(It, DoneMBB);

// Transfer the remainder of BB and its successor edges to DoneMBB.
DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
DoneMBB->transferSuccessorsAndUpdatePHIs(BB);

BB->addSuccessor(TrueMBB);
BB->addSuccessor(FalseMBB);
TrueMBB->addSuccessor(DoneMBB);
FalseMBB->addSuccessor(DoneMBB);

unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));

MI.eraseFromParent();
// For signed numbers, we can do a single comparison to determine whether
// fabs(x) is within range.
if (IsUnsigned) {
  Tmp0 = InReg;
} else {
  BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
}
BuildMI(BB, DL, TII.get(FConst), Tmp1)
    .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);

// For unsigned numbers, we have to do a separate comparison with zero.
if (IsUnsigned) {
  Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
  Register SecondCmpReg =
      MRI.createVirtualRegister(&WebAssembly::I32RegClass);
  Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
  BuildMI(BB, DL, TII.get(FConst), Tmp1)
      .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
  BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
  BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
  CmpReg = AndReg;
}

BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);

// Create the CFG diamond to select between doing the conversion or using
// the substitute value.
BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
    .addReg(FalseReg)
    .addMBB(FalseMBB)
    .addReg(TrueReg)
    .addMBB(TrueMBB);

return DoneMBB;
497}

499static MachineBasicBlock *
500LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
               const WebAssemblySubtarget *Subtarget,
               const TargetInstrInfo &TII) {
MachineInstr &CallParams = *CallResults.getPrevNode();
assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS)(static_cast <bool> (CallParams.getOpcode() == WebAssembly
::CALL_PARAMS) ? void (0) : __assert_fail ("CallParams.getOpcode() == WebAssembly::CALL_PARAMS"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 504, __extension__ __PRETTY_FUNCTION__));
assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 506, __extension__ __PRETTY_FUNCTION__))
       CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS)(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 506, __extension__ __PRETTY_FUNCTION__));

bool IsIndirect = CallParams.getOperand(0).isReg();
bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;

bool IsFuncrefCall = false;
if (IsIndirect) {
  Register Reg = CallParams.getOperand(0).getReg();
  const MachineFunction *MF = BB->getParent();
  const MachineRegisterInfo &MRI = MF->getRegInfo();
  const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
  IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
  assert(!IsFuncrefCall || Subtarget->hasReferenceTypes())(static_cast <bool> (!IsFuncrefCall || Subtarget->hasReferenceTypes
()) ? void (0) : __assert_fail ("!IsFuncrefCall || Subtarget->hasReferenceTypes()"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 518, __extension__ __PRETTY_FUNCTION__));
}

unsigned CallOp;
if (IsIndirect && IsRetCall) {
  CallOp = WebAssembly::RET_CALL_INDIRECT;
} else if (IsIndirect) {
  CallOp = WebAssembly::CALL_INDIRECT;
} else if (IsRetCall) {
  CallOp = WebAssembly::RET_CALL;
} else {
  CallOp = WebAssembly::CALL;
}

MachineFunction &MF = *BB->getParent();
const MCInstrDesc &MCID = TII.get(CallOp);
MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));

// See if we must truncate the function pointer.
// CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers
// as 64-bit for uniformity with other pointer types.
// See also: WebAssemblyFastISel::selectCall
if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) {
  Register Reg32 =
      MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
  auto &FnPtr = CallParams.getOperand(0);
  BuildMI(*BB, CallResults.getIterator(), DL,
          TII.get(WebAssembly::I32_WRAP_I64), Reg32)
      .addReg(FnPtr.getReg());
  FnPtr.setReg(Reg32);
}

// Move the function pointer to the end of the arguments for indirect calls
if (IsIndirect) {
  auto FnPtr = CallParams.getOperand(0);
  CallParams.RemoveOperand(0);
  CallParams.addOperand(FnPtr);
}

for (auto Def : CallResults.defs())
  MIB.add(Def);

if (IsIndirect) {
  // Placeholder for the type index.
  MIB.addImm(0);
  // The table into which this call_indirect indexes.
  MCSymbolWasm *Table = IsFuncrefCall
                            ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
                                  MF.getContext(), Subtarget)
                            : WebAssembly::getOrCreateFunctionTableSymbol(
                                  MF.getContext(), Subtarget);
  if (Subtarget->hasReferenceTypes()) {
    MIB.addSym(Table);
  } else {
    // For the MVP there is at most one table whose number is 0, but we can't
    // write a table symbol or issue relocations.  Instead we just ensure the
    // table is live and write a zero.
    Table->setNoStrip();
    MIB.addImm(0);
  }
}

for (auto Use : CallParams.uses())
  MIB.add(Use);

BB->insert(CallResults.getIterator(), MIB);
CallParams.eraseFromParent();
CallResults.eraseFromParent();

// If this is a funcref call, to avoid hidden GC roots, we need to clear the
// table slot with ref.null upon call_indirect return.
//
// This generates the following code, which comes right after a call_indirect
// of a funcref:
//
//    i32.const 0
//    ref.null func
//    table.set __funcref_call_table
if (IsIndirect && IsFuncrefCall) {
  MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
      MF.getContext(), Subtarget);
  Register RegZero =
      MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
  MachineInstr *Const0 =
      BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
  BB->insertAfter(MIB.getInstr()->getIterator(), Const0);

  Register RegFuncref =
      MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
  MachineInstr *RefNull =
      BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref)
          .addImm(static_cast<int32_t>(WebAssembly::HeapType::Funcref));
  BB->insertAfter(Const0->getIterator(), RefNull);

  MachineInstr *TableSet =
      BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
          .addSym(Table)
          .addReg(RegZero)
          .addReg(RegFuncref);
  BB->insertAfter(RefNull->getIterator(), TableSet);
}

return BB;
621}

623MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
  MachineInstr &MI, MachineBasicBlock *BB) const {
const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
DebugLoc DL = MI.getDebugLoc();

switch (MI.getOpcode()) {
default:
  llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 630);
case WebAssembly::FP_TO_SINT_I32_F32:
  return LowerFPToInt(MI, DL, BB, TII, false, false, false,
                      WebAssembly::I32_TRUNC_S_F32);
case WebAssembly::FP_TO_UINT_I32_F32:
  return LowerFPToInt(MI, DL, BB, TII, true, false, false,
                      WebAssembly::I32_TRUNC_U_F32);
case WebAssembly::FP_TO_SINT_I64_F32:
  return LowerFPToInt(MI, DL, BB, TII, false, true, false,
                      WebAssembly::I64_TRUNC_S_F32);
case WebAssembly::FP_TO_UINT_I64_F32:
  return LowerFPToInt(MI, DL, BB, TII, true, true, false,
                      WebAssembly::I64_TRUNC_U_F32);
case WebAssembly::FP_TO_SINT_I32_F64:
  return LowerFPToInt(MI, DL, BB, TII, false, false, true,
                      WebAssembly::I32_TRUNC_S_F64);
case WebAssembly::FP_TO_UINT_I32_F64:
  return LowerFPToInt(MI, DL, BB, TII, true, false, true,
                      WebAssembly::I32_TRUNC_U_F64);
case WebAssembly::FP_TO_SINT_I64_F64:
  return LowerFPToInt(MI, DL, BB, TII, false, true, true,
                      WebAssembly::I64_TRUNC_S_F64);
case WebAssembly::FP_TO_UINT_I64_F64:
  return LowerFPToInt(MI, DL, BB, TII, true, true, true,
                      WebAssembly::I64_TRUNC_U_F64);
case WebAssembly::CALL_RESULTS:
case WebAssembly::RET_CALL_RESULTS:
  return LowerCallResults(MI, DL, BB, Subtarget, TII);
}
659}

661const char *
662WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
case WebAssemblyISD::FIRST_NUMBER:
case WebAssemblyISD::FIRST_MEM_OPCODE:
  break;
667#define HANDLE_NODETYPE(NODE)                                                  \
case WebAssemblyISD::NODE:                                                   \
  return "WebAssemblyISD::" #NODE;
670#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
671#include "WebAssemblyISD.def"
672#undef HANDLE_MEM_NODETYPE
673#undef HANDLE_NODETYPE
}
return nullptr;
676}

678std::pair<unsigned, const TargetRegisterClass *>
679WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
  const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
// First, see if this is a constraint that directly corresponds to a
// WebAssembly register class.
if (Constraint.size() == 1) {
  switch (Constraint[0]) {
  case 'r':
    assert(VT != MVT::iPTR && "Pointer MVT not expected here")(static_cast <bool> (VT != MVT::iPTR && "Pointer MVT not expected here"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 686, __extension__ __PRETTY_FUNCTION__));
    if (Subtarget->hasSIMD128() && VT.isVector()) {
      if (VT.getSizeInBits() == 128)
        return std::make_pair(0U, &WebAssembly::V128RegClass);
    }
    if (VT.isInteger() && !VT.isVector()) {
      if (VT.getSizeInBits() <= 32)
        return std::make_pair(0U, &WebAssembly::I32RegClass);
      if (VT.getSizeInBits() <= 64)
        return std::make_pair(0U, &WebAssembly::I64RegClass);
    }
    if (VT.isFloatingPoint() && !VT.isVector()) {
      switch (VT.getSizeInBits()) {
      case 32:
        return std::make_pair(0U, &WebAssembly::F32RegClass);
      case 64:
        return std::make_pair(0U, &WebAssembly::F64RegClass);
      default:
        break;
      }
    }
    break;
  default:
    break;
  }
}

return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
714}

716bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const {
// Assume ctz is a relatively cheap operation.
return true;
719}

721bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
// Assume clz is a relatively cheap operation.
return true;
724}

726bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                                    const AddrMode &AM,
                                                    Type *Ty, unsigned AS,
                                                    Instruction *I) const {
// WebAssembly offsets are added as unsigned without wrapping. The
// isLegalAddressingMode gives us no way to determine if wrapping could be
// happening, so we approximate this by accepting only non-negative offsets.
if (AM.BaseOffs < 0)
  return false;

// WebAssembly has no scale register operands.
if (AM.Scale != 0)
  return false;

// Everything else is legal.
return true;
742}

744bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
  EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
  MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
// WebAssembly supports unaligned accesses, though it should be declared
// with the p2align attribute on loads and stores which do so, and there
// may be a performance impact. We tell LLVM they're "fast" because
// for the kinds of things that LLVM uses this for (merging adjacent stores
// of constants, etc.), WebAssembly implementations will either want the
// unaligned access or they'll split anyway.
if (Fast)
  *Fast = true;
return true;
756}

758bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
                                            AttributeList Attr) const {
// The current thinking is that wasm engines will perform this optimization,
// so we can save on code size.
return true;
763}

765bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
EVT ExtT = ExtVal.getValueType();
EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
       (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
       (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
771}

773EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
                                                LLVMContext &C,
                                                EVT VT) const {
if (VT.isVector())
  return VT.changeVectorElementTypeToInteger();

// So far, all branch instructions in Wasm take an I32 condition.
// The default TargetLowering::getSetCCResultType returns the pointer size,
// which would be useful to reduce instruction counts when testing
// against 64-bit pointers/values if at some point Wasm supports that.
return EVT::getIntegerVT(C, 32);
784}

786bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                                 const CallInst &I,
                                                 MachineFunction &MF,
                                                 unsigned Intrinsic) const {
switch (Intrinsic) {
case Intrinsic::wasm_memory_atomic_notify:
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::i32;
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(4);
  // atomic.notify instruction does not really load the memory specified with
  // this argument, but MachineMemOperand should either be load or store, so
  // we set this to a load.
  // FIXME Volatile isn't really correct, but currently all LLVM atomic
  // instructions are treated as volatiles in the backend, so we should be
  // consistent. The same applies for wasm_atomic_wait intrinsics too.
  Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
  return true;
case Intrinsic::wasm_memory_atomic_wait32:
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::i32;
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(4);
  Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
  return true;
case Intrinsic::wasm_memory_atomic_wait64:
  Info.opc = ISD::INTRINSIC_W_CHAIN;
  Info.memVT = MVT::i64;
  Info.ptrVal = I.getArgOperand(0);
  Info.offset = 0;
  Info.align = Align(8);
  Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
  return true;
default:
  return false;
}
824}

826//===----------------------------------------------------------------------===//
827// WebAssembly Lowering private implementation.
828//===----------------------------------------------------------------------===//

830//===----------------------------------------------------------------------===//
831// Lowering Code
832//===----------------------------------------------------------------------===//

834static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
MachineFunction &MF = DAG.getMachineFunction();
DAG.getContext()->diagnose(
    DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
838}

840// Test whether the given calling convention is supported.
841static bool callingConvSupported(CallingConv::ID CallConv) {
// We currently support the language-independent target-independent
// conventions. We don't yet have a way to annotate calls with properties like
// "cold", and we don't have any call-clobbered registers, so these are mostly
// all handled the same.
return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
       CallConv == CallingConv::Cold ||
       CallConv == CallingConv::PreserveMost ||
       CallConv == CallingConv::PreserveAll ||
       CallConv == CallingConv::CXX_FAST_TLS ||
       CallConv == CallingConv::WASM_EmscriptenInvoke ||
       CallConv == CallingConv::Swift;
853}

855SDValue
856WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
                                   SmallVectorImpl<SDValue> &InVals) const {
SelectionDAG &DAG = CLI.DAG;
SDLoc DL = CLI.DL;
SDValue Chain = CLI.Chain;
SDValue Callee = CLI.Callee;
MachineFunction &MF = DAG.getMachineFunction();
auto Layout = MF.getDataLayout();

CallingConv::ID CallConv = CLI.CallConv;
if (!callingConvSupported(CallConv))
  fail(DL, DAG,
       "WebAssembly doesn't support language-specific or target-specific "
       "calling conventions yet");
if (CLI.IsPatchPoint)
  fail(DL, DAG, "WebAssembly doesn't support patch point yet");

if (CLI.IsTailCall) {
  auto NoTail = [&](const char *Msg) {
    if (CLI.CB && CLI.CB->isMustTailCall())
      fail(DL, DAG, Msg);
    CLI.IsTailCall = false;
  };

  if (!Subtarget->hasTailCall())
    NoTail("WebAssembly 'tail-call' feature not enabled");

  // Varargs calls cannot be tail calls because the buffer is on the stack
  if (CLI.IsVarArg)
    NoTail("WebAssembly does not support varargs tail calls");

  // Do not tail call unless caller and callee return types match
  const Function &F = MF.getFunction();
  const TargetMachine &TM = getTargetMachine();
  Type *RetTy = F.getReturnType();
  SmallVector<MVT, 4> CallerRetTys;
  SmallVector<MVT, 4> CalleeRetTys;
  computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
  computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
  bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
                    std::equal(CallerRetTys.begin(), CallerRetTys.end(),
                               CalleeRetTys.begin());
  if (!TypesMatch)
    NoTail("WebAssembly tail call requires caller and callee return types to "
           "match");

  // If pointers to local stack values are passed, we cannot tail call
  if (CLI.CB) {
    for (auto &Arg : CLI.CB->args()) {
      Value *Val = Arg.get();
      // Trace the value back through pointer operations
      while (true) {
        Value *Src = Val->stripPointerCastsAndAliases();
        if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
          Src = GEP->getPointerOperand();
        if (Val == Src)
          break;
        Val = Src;
      }
      if (isa<AllocaInst>(Val)) {
        NoTail(
            "WebAssembly does not support tail calling with stack arguments");
        break;
      }
    }
  }
}

SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;

// The generic code may have added an sret argument. If we're lowering an
// invoke function, the ABI requires that the function pointer be the first
// argument, so we may have to swap the arguments.
if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
    Outs[0].Flags.isSRet()) {
  std::swap(Outs[0], Outs[1]);
  std::swap(OutVals[0], OutVals[1]);
}

bool HasSwiftSelfArg = false;
bool HasSwiftErrorArg = false;
unsigned NumFixedArgs = 0;
for (unsigned I = 0; I < Outs.size(); ++I) {
  const ISD::OutputArg &Out = Outs[I];
  SDValue &OutVal = OutVals[I];
  HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
  HasSwiftErrorArg |= Out.Flags.isSwiftError();
  if (Out.Flags.isNest())
    fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
  if (Out.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
  if (Out.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
  if (Out.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
  if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
    auto &MFI = MF.getFrameInfo();
    int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
                                   Out.Flags.getNonZeroByValAlign(),
                                   /*isSS=*/false);
    SDValue SizeNode =
        DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
    SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
    Chain = DAG.getMemcpy(
        Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
        /*isVolatile*/ false, /*AlwaysInline=*/false,
        /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
    OutVal = FINode;
  }
  // Count the number of fixed args *after* legalization.
  NumFixedArgs += Out.IsFixed;
}

bool IsVarArg = CLI.IsVarArg;
auto PtrVT = getPointerTy(Layout);

// For swiftcc, emit additional swiftself and swifterror arguments
// if there aren't. These additional arguments are also added for callee
// signature They are necessary to match callee and caller signature for
// indirect call.
if (CallConv == CallingConv::Swift) {
  if (!HasSwiftSelfArg) {
    NumFixedArgs++;
    ISD::OutputArg Arg;
    Arg.Flags.setSwiftSelf();
    CLI.Outs.push_back(Arg);
    SDValue ArgVal = DAG.getUNDEF(PtrVT);
    CLI.OutVals.push_back(ArgVal);
  }
  if (!HasSwiftErrorArg) {
    NumFixedArgs++;
    ISD::OutputArg Arg;
    Arg.Flags.setSwiftError();
    CLI.Outs.push_back(Arg);
    SDValue ArgVal = DAG.getUNDEF(PtrVT);
    CLI.OutVals.push_back(ArgVal);
  }
}

// Analyze operands of the call, assigning locations to each operand.
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());

if (IsVarArg) {
  // Outgoing non-fixed arguments are placed in a buffer. First
  // compute their offsets and the total amount of buffer space needed.
  for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
    const ISD::OutputArg &Out = Outs[I];
    SDValue &Arg = OutVals[I];
    EVT VT = Arg.getValueType();
    assert(VT != MVT::iPTR && "Legalized args should be concrete")(static_cast <bool> (VT != MVT::iPTR && "Legalized args should be concrete"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1008, __extension__ __PRETTY_FUNCTION__));
    Type *Ty = VT.getTypeForEVT(*DAG.getContext());
    Align Alignment =
        std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
    unsigned Offset =
        CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
    CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
                                      Offset, VT.getSimpleVT(),
                                      CCValAssign::Full));
  }
}

unsigned NumBytes = CCInfo.getAlignedCallFrameSize();

SDValue FINode;
if (IsVarArg && NumBytes) {
  // For non-fixed arguments, next emit stores to store the argument values
  // to the stack buffer at the offsets computed above.
  int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
                                               Layout.getStackAlignment(),
                                               /*isSS=*/false);
  unsigned ValNo = 0;
  SmallVector<SDValue, 8> Chains;
  for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
    assert(ArgLocs[ValNo].getValNo() == ValNo &&(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
 && "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1033, __extension__ __PRETTY_FUNCTION__))
           "ArgLocs should remain in order and only hold varargs args")(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
 && "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1033, __extension__ __PRETTY_FUNCTION__));
    unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
    FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
    SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
                              DAG.getConstant(Offset, DL, PtrVT));
    Chains.push_back(
        DAG.getStore(Chain, DL, Arg, Add,
                     MachinePointerInfo::getFixedStack(MF, FI, Offset)));
  }
  if (!Chains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
} else if (IsVarArg) {
  FINode = DAG.getIntPtrConstant(0, DL);
}

if (Callee->getOpcode() == ISD::GlobalAddress) {
  // If the callee is a GlobalAddress node (quite common, every direct call
  // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
  // doesn't at MO_GOT which is not needed for direct calls.
  GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee);
  Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
                                      getPointerTy(DAG.getDataLayout()),
                                      GA->getOffset());
  Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
                       getPointerTy(DAG.getDataLayout()), Callee);
}

// Compute the operands for the CALLn node.
SmallVector<SDValue, 16> Ops;
Ops.push_back(Chain);
Ops.push_back(Callee);

// Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
// isn't reliable.
Ops.append(OutVals.begin(),
           IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
// Add a pointer to the vararg buffer.
if (IsVarArg)
  Ops.push_back(FINode);

SmallVector<EVT, 8> InTys;
for (const auto &In : Ins) {
  assert(!In.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!In.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1075, __extension__ __PRETTY_FUNCTION__));
  assert(!In.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!In.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1076, __extension__ __PRETTY_FUNCTION__));
  if (In.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
  if (In.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
  if (In.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG,
         "WebAssembly hasn't implemented cons regs last return values");
  // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
  // registers.
  InTys.push_back(In.VT);
}

// Lastly, if this is a call to a funcref we need to add an instruction
// table.set to the chain and transform the call.
if (CLI.CB && isFuncrefType(CLI.CB->getCalledOperand()->getType())) {
  // In the absence of function references proposal where a funcref call is
  // lowered to call_ref, using reference types we generate a table.set to set
  // the funcref to a special table used solely for this purpose, followed by
  // a call_indirect. Here we just generate the table set, and return the
  // SDValue of the table.set so that LowerCall can finalize the lowering by
  // generating the call_indirect.
  SDValue Chain = Ops[0];

  MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
      MF.getContext(), Subtarget);
  SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
  SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
  SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
  SDValue TableSet = DAG.getMemIntrinsicNode(
      WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
      MVT::funcref,
      // Machine Mem Operand args
      MachinePointerInfo(WasmAddressSpace::FUNCREF),
      CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
      MachineMemOperand::MOStore);

  Ops[0] = TableSet; // The new chain is the TableSet itself
}

if (CLI.IsTailCall) {
  // ret_calls do not return values to the current frame
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
  return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
}

InTys.push_back(MVT::Other);
SDVTList InTyList = DAG.getVTList(InTys);
SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);

for (size_t I = 0; I < Ins.size(); ++I)
  InVals.push_back(Res.getValue(I));

// Return the chain
return Res.getValue(Ins.size());
1131}

1133bool WebAssemblyTargetLowering::CanLowerReturn(
  CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
  const SmallVectorImpl<ISD::OutputArg> &Outs,
  LLVMContext & /*Context*/) const {
// WebAssembly can only handle returning tuples with multivalue enabled
return Subtarget->hasMultivalue() || Outs.size() <= 1;
1139}

1141SDValue WebAssemblyTargetLowering::LowerReturn(
  SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
  const SmallVectorImpl<ISD::OutputArg> &Outs,
  const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
  SelectionDAG &DAG) const {
assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1147, __extension__ __PRETTY_FUNCTION__))
       "MVP WebAssembly can only return up to one value")(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1147, __extension__ __PRETTY_FUNCTION__));
if (!callingConvSupported(CallConv))
  fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");

SmallVector<SDValue, 4> RetOps(1, Chain);
RetOps.append(OutVals.begin(), OutVals.end());
Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);

// Record the number and types of the return values.
for (const ISD::OutputArg &Out : Outs) {
  assert(!Out.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!Out.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1157, __extension__ __PRETTY_FUNCTION__));
  assert(!Out.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!Out.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1158, __extension__ __PRETTY_FUNCTION__));
  assert(Out.IsFixed && "non-fixed return value is not valid")(static_cast <bool> (Out.IsFixed && "non-fixed return value is not valid"
) ? void (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1159, __extension__ __PRETTY_FUNCTION__));
  if (Out.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
  if (Out.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
  if (Out.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
}

return Chain;
1169}

1171SDValue WebAssemblyTargetLowering::LowerFormalArguments(
  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
if (!callingConvSupported(CallConv))
  fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");

MachineFunction &MF = DAG.getMachineFunction();
auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();

// Set up the incoming ARGUMENTS value, which serves to represent the liveness
// of the incoming values before they're represented by virtual registers.
MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);

bool HasSwiftErrorArg = false;
bool HasSwiftSelfArg = false;
for (const ISD::InputArg &In : Ins) {
  HasSwiftSelfArg |= In.Flags.isSwiftSelf();
  HasSwiftErrorArg |= In.Flags.isSwiftError();
  if (In.Flags.isInAlloca())
    fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
  if (In.Flags.isNest())
    fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
  if (In.Flags.isInConsecutiveRegs())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
  if (In.Flags.isInConsecutiveRegsLast())
    fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
  // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
  // registers.
  InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
                                         DAG.getTargetConstant(InVals.size(),
                                                               DL, MVT::i32))
                           : DAG.getUNDEF(In.VT));

  // Record the number and types of arguments.
  MFI->addParam(In.VT);
}

// For swiftcc, emit additional swiftself and swifterror arguments
// if there aren't. These additional arguments are also added for callee
// signature They are necessary to match callee and caller signature for
// indirect call.
auto PtrVT = getPointerTy(MF.getDataLayout());
if (CallConv == CallingConv::Swift) {
  if (!HasSwiftSelfArg) {
    MFI->addParam(PtrVT);
  }
  if (!HasSwiftErrorArg) {
    MFI->addParam(PtrVT);
  }
}
// Varargs are copied into a buffer allocated by the caller, and a pointer to
// the buffer is passed as an argument.
if (IsVarArg) {
  MVT PtrVT = getPointerTy(MF.getDataLayout());
  Register VarargVreg =
      MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
  MFI->setVarargBufferVreg(VarargVreg);
  Chain = DAG.getCopyToReg(
      Chain, DL, VarargVreg,
      DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
                  DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
  MFI->addParam(PtrVT);
}

// Record the number and types of arguments and results.
SmallVector<MVT, 4> Params;
SmallVector<MVT, 4> Results;
computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(),
                    MF.getFunction(), DAG.getTarget(), Params, Results);
for (MVT VT : Results)
  MFI->addResult(VT);
// TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
// the param logic here with ComputeSignatureVTs
assert(MFI->getParams().size() == Params.size() &&(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
 ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1247, __extension__ __PRETTY_FUNCTION__))
       std::equal(MFI->getParams().begin(), MFI->getParams().end(),(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
 ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1247, __extension__ __PRETTY_FUNCTION__))
                  Params.begin()))(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
 ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1247, __extension__ __PRETTY_FUNCTION__));

return Chain;
1250}

1252void WebAssemblyTargetLowering::ReplaceNodeResults(
  SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
switch (N->getOpcode()) {
case ISD::SIGN_EXTEND_INREG:
  // Do not add any results, signifying that N should not be custom lowered
  // after all. This happens because simd128 turns on custom lowering for
  // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
  // illegal type.
  break;
default:
  llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1263)
      "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1263);
}
1265}

1267//===----------------------------------------------------------------------===//
1268//  Custom lowering hooks.
1269//===----------------------------------------------------------------------===//

1271SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
                                                SelectionDAG &DAG) const {
SDLoc DL(Op);
switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:'  at line 1311→
default:
  llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1276);
  return SDValue();
case ISD::FrameIndex:
  return LowerFrameIndex(Op, DAG);
case ISD::GlobalAddress:
  return LowerGlobalAddress(Op, DAG);
case ISD::GlobalTLSAddress:
  return LowerGlobalTLSAddress(Op, DAG);
case ISD::ExternalSymbol:
  return LowerExternalSymbol(Op, DAG);
case ISD::JumpTable:
  return LowerJumpTable(Op, DAG);
case ISD::BR_JT:
  return LowerBR_JT(Op, DAG);
case ISD::VASTART:
  return LowerVASTART(Op, DAG);
case ISD::BlockAddress:
case ISD::BRIND:
  fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
  return SDValue();
case ISD::RETURNADDR:
  return LowerRETURNADDR(Op, DAG);
case ISD::FRAMEADDR:
  return LowerFRAMEADDR(Op, DAG);
case ISD::CopyToReg:
  return LowerCopyToReg(Op, DAG);
case ISD::EXTRACT_VECTOR_ELT:
case ISD::INSERT_VECTOR_ELT:
  return LowerAccessVectorElement(Op, DAG);
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_WO_CHAIN:
case ISD::INTRINSIC_W_CHAIN:
  return LowerIntrinsic(Op, DAG);
case ISD::SIGN_EXTEND_INREG:
  return LowerSIGN_EXTEND_INREG(Op, DAG);
case ISD::BUILD_VECTOR:
  return LowerBUILD_VECTOR(Op, DAG);
2
←
Calling 'WebAssemblyTargetLowering::LowerBUILD_VECTOR'→
case ISD::VECTOR_SHUFFLE:
  return LowerVECTOR_SHUFFLE(Op, DAG);
case ISD::SETCC:
  return LowerSETCC(Op, DAG);
case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
  return LowerShift(Op, DAG);
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
  return LowerFP_TO_INT_SAT(Op, DAG);
case ISD::LOAD:
  return LowerLoad(Op, DAG);
case ISD::STORE:
  return LowerStore(Op, DAG);
}
1329}

1331static bool IsWebAssemblyGlobal(SDValue Op) {
if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
  return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace());

return false;
1336}

1338static Optional<unsigned> IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG) {
const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
if (!FI)
  return None;

auto &MF = DAG.getMachineFunction();
return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex());
1345}

1347bool WebAssemblyTargetLowering::isFuncrefType(const Type *Ty) {
return isa<PointerType>(Ty) &&
       Ty->getPointerAddressSpace() == WasmAddressSpace::FUNCREF;
1350}

1352bool WebAssemblyTargetLowering::isExternrefType(const Type *Ty) {
return isa<PointerType>(Ty) &&
       Ty->getPointerAddressSpace() == WasmAddressSpace::EXTERNREF;
1355}

1357SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
const SDValue &Value = SN->getValue();
const SDValue &Base = SN->getBasePtr();
const SDValue &Offset = SN->getOffset();

if (IsWebAssemblyGlobal(Base)) {
  if (!Offset->isUndef())
    report_fatal_error("unexpected offset when storing to webassembly global",
                       false);

  SDVTList Tys = DAG.getVTList(MVT::Other);
  SDValue Ops[] = {SN->getChain(), Value, Base};
  return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
                                 SN->getMemoryVT(), SN->getMemOperand());
}

if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
  if (!Offset->isUndef())
    report_fatal_error("unexpected offset when storing to webassembly local",
                       false);

  SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
  SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
  SDValue Ops[] = {SN->getChain(), Idx, Value};
  return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
}

return Op;
1388}

1390SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
                                           SelectionDAG &DAG) const {
SDLoc DL(Op);
LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
const SDValue &Base = LN->getBasePtr();
const SDValue &Offset = LN->getOffset();

if (IsWebAssemblyGlobal(Base)) {
  if (!Offset->isUndef())
    report_fatal_error(
        "unexpected offset when loading from webassembly global", false);

  SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
  SDValue Ops[] = {LN->getChain(), Base};
  return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
                                 LN->getMemoryVT(), LN->getMemOperand());
}

if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
  if (!Offset->isUndef())
    report_fatal_error(
        "unexpected offset when loading from webassembly local", false);

  SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
  EVT LocalVT = LN->getValueType(0);
  SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
                                 {LN->getChain(), Idx});
  SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
  assert(Result->getNumValues() == 2 && "Loads must carry a chain!")(static_cast <bool> (Result->getNumValues() == 2 &&
 "Loads must carry a chain!") ? void (0) : __assert_fail ("Result->getNumValues() == 2 && \"Loads must carry a chain!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1418, __extension__ __PRETTY_FUNCTION__));
  return Result;
}

return Op;
1423}

1425SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
                                                SelectionDAG &DAG) const {
SDValue Src = Op.getOperand(2);
if (isa<FrameIndexSDNode>(Src.getNode())) {
  // CopyToReg nodes don't support FrameIndex operands. Other targets select
  // the FI to some LEA-like instruction, but since we don't have that, we
  // need to insert some kind of instruction that can take an FI operand and
  // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
  // local.copy between Op and its FI operand.
  SDValue Chain = Op.getOperand(0);
  SDLoc DL(Op);
  unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
  EVT VT = Src.getValueType();
  SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
                                                 : WebAssembly::COPY_I64,
                                  DL, VT, Src),
               0);
  return Op.getNode()->getNumValues() == 1
             ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
             : DAG.getCopyToReg(Chain, DL, Reg, Copy,
                                Op.getNumOperands() == 4 ? Op.getOperand(3)
                                                         : SDValue());
}
return SDValue();
1449}

1451SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
                                                 SelectionDAG &DAG) const {
int FI = cast<FrameIndexSDNode>(Op)->getIndex();
return DAG.getTargetFrameIndex(FI, Op.getValueType());
1455}

1457SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
                                                 SelectionDAG &DAG) const {
SDLoc DL(Op);

if (!Subtarget->getTargetTriple().isOSEmscripten()) {
  fail(DL, DAG,
       "Non-Emscripten WebAssembly hasn't implemented "
       "__builtin_return_address");
  return SDValue();
}

if (verifyReturnAddressArgumentIsConstant(Op, DAG))
  return SDValue();

unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
MakeLibCallOptions CallOptions;
return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
                   {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
    .first;
1476}

1478SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
                                                SelectionDAG &DAG) const {
// Non-zero depths are not supported by WebAssembly currently. Use the
// legalizer's default expansion, which is to return 0 (what this function is
// documented to do).
if (Op.getConstantOperandVal(0) > 0)
  return SDValue();

DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
EVT VT = Op.getValueType();
Register FP =
    Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1491}

1493SDValue
1494WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
                                               SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *GA = cast<GlobalAddressSDNode>(Op);
MVT PtrVT = getPointerTy(DAG.getDataLayout());

MachineFunction &MF = DAG.getMachineFunction();
if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
  report_fatal_error("cannot use thread-local storage without bulk memory",
                     false);

const GlobalValue *GV = GA->getGlobal();

// Currently Emscripten does not support dynamic linking with threads.
// Therefore, if we have thread-local storage, only the local-exec model
// is possible.
// TODO: remove this and implement proper TLS models once Emscripten
// supports dynamic linking with threads.
if (GV->getThreadLocalMode() != GlobalValue::LocalExecTLSModel &&
    !Subtarget->getTargetTriple().isOSEmscripten()) {
  report_fatal_error("only -ftls-model=local-exec is supported for now on "
                     "non-Emscripten OSes: variable " +
                         GV->getName(),
                     false);
}

auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
                                   : WebAssembly::GLOBAL_GET_I32;
const char *BaseName = MF.createExternalSymbolName("__tls_base");

SDValue BaseAddr(
    DAG.getMachineNode(GlobalGet, DL, PtrVT,
                       DAG.getTargetExternalSymbol(BaseName, PtrVT)),
    0);

SDValue TLSOffset = DAG.getTargetGlobalAddress(
    GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
SDValue SymAddr = DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, TLSOffset);

return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
1534}

1536SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
                                                    SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *GA = cast<GlobalAddressSDNode>(Op);
EVT VT = Op.getValueType();
assert(GA->getTargetFlags() == 0 &&(static_cast <bool> (GA->getTargetFlags() == 0 &&
 "Unexpected target flags on generic GlobalAddressSDNode") ? void
 (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1542, __extension__ __PRETTY_FUNCTION__))
       "Unexpected target flags on generic GlobalAddressSDNode")(static_cast <bool> (GA->getTargetFlags() == 0 &&
 "Unexpected target flags on generic GlobalAddressSDNode") ? void
 (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1542, __extension__ __PRETTY_FUNCTION__));
if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace()))
  fail(DL, DAG, "Invalid address space for WebAssembly target");

unsigned OperandFlags = 0;
if (isPositionIndependent()) {
  const GlobalValue *GV = GA->getGlobal();
  if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
    MachineFunction &MF = DAG.getMachineFunction();
    MVT PtrVT = getPointerTy(MF.getDataLayout());
    const char *BaseName;
    if (GV->getValueType()->isFunctionTy()) {
      BaseName = MF.createExternalSymbolName("__table_base");
      OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
    }
    else {
      BaseName = MF.createExternalSymbolName("__memory_base");
      OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
    }
    SDValue BaseAddr =
        DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
                    DAG.getTargetExternalSymbol(BaseName, PtrVT));

    SDValue SymAddr = DAG.getNode(
        WebAssemblyISD::WrapperPIC, DL, VT,
        DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
                                   OperandFlags));

    return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
  } else {
    OperandFlags = WebAssemblyII::MO_GOT;
  }
}

return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                   DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
                                              GA->getOffset(), OperandFlags));
1579}

1581SDValue
1582WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
                                             SelectionDAG &DAG) const {
SDLoc DL(Op);
const auto *ES = cast<ExternalSymbolSDNode>(Op);
EVT VT = Op.getValueType();
assert(ES->getTargetFlags() == 0 &&(static_cast <bool> (ES->getTargetFlags() == 0 &&
 "Unexpected target flags on generic ExternalSymbolSDNode") ?
 void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1588, __extension__ __PRETTY_FUNCTION__))
       "Unexpected target flags on generic ExternalSymbolSDNode")(static_cast <bool> (ES->getTargetFlags() == 0 &&
 "Unexpected target flags on generic ExternalSymbolSDNode") ?
 void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1588, __extension__ __PRETTY_FUNCTION__));
return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                   DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1591}

1593SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
                                                SelectionDAG &DAG) const {
// There's no need for a Wrapper node because we always incorporate a jump
// table operand into a BR_TABLE instruction, rather than ever
// materializing it in a register.
const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
                              JT->getTargetFlags());
1601}

1603SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue Chain = Op.getOperand(0);
const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
SDValue Index = Op.getOperand(2);
assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")(static_cast <bool> (JT->getTargetFlags() == 0 &&
 "WebAssembly doesn't set target flags") ? void (0) : __assert_fail
 ("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1609, __extension__ __PRETTY_FUNCTION__));

SmallVector<SDValue, 8> Ops;
Ops.push_back(Chain);
Ops.push_back(Index);

MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;

// Add an operand for each case.
for (auto MBB : MBBs)
  Ops.push_back(DAG.getBasicBlock(MBB));

// Add the first MBB as a dummy default target for now. This will be replaced
// with the proper default target (and the preceding range check eliminated)
// if possible by WebAssemblyFixBrTableDefaults.
Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1627}

1629SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
                                              SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());

auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();

SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
                                  MFI->getVarargBufferVreg(), PtrVT);
return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
                    MachinePointerInfo(SV));
1641}

1643static SDValue getCppExceptionSymNode(SDValue Op, unsigned TagIndex,
                                    SelectionDAG &DAG) {
// We only support C++ exceptions for now
int Tag =
    cast<ConstantSDNode>(Op.getOperand(TagIndex).getNode())->getZExtValue();
if (Tag != WebAssembly::CPP_EXCEPTION)
  llvm_unreachable("Invalid tag: We only support C++ exceptions for now")::llvm::llvm_unreachable_internal("Invalid tag: We only support C++ exceptions for now"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1649);
auto &MF = DAG.getMachineFunction();
const auto &TLI = DAG.getTargetLoweringInfo();
MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
const char *SymName = MF.createExternalSymbolName("__cpp_exception");
return DAG.getNode(WebAssemblyISD::Wrapper, SDLoc(Op), PtrVT,
                   DAG.getTargetExternalSymbol(SymName, PtrVT));
1656}

1658SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
                                                SelectionDAG &DAG) const {
MachineFunction &MF = DAG.getMachineFunction();
unsigned IntNo;
switch (Op.getOpcode()) {
case ISD::INTRINSIC_VOID:
case ISD::INTRINSIC_W_CHAIN:
  IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
  break;
case ISD::INTRINSIC_WO_CHAIN:
  IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
  break;
default:
  llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1671);
}
SDLoc DL(Op);

switch (IntNo) {
default:
  return SDValue(); // Don't custom lower most intrinsics.

case Intrinsic::wasm_lsda: {
  EVT VT = Op.getValueType();
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
  auto &Context = MF.getMMI().getContext();
  MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
                                          Twine(MF.getFunctionNumber()));
  return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                     DAG.getMCSymbol(S, PtrVT));
}

case Intrinsic::wasm_throw: {
  SDValue SymNode = getCppExceptionSymNode(Op, 2, DAG);
  return DAG.getNode(WebAssemblyISD::THROW, DL,
                     MVT::Other, // outchain type
                     {
                         Op.getOperand(0), // inchain
                         SymNode,          // exception symbol
                         Op.getOperand(3)  // thrown value
                     });
}

case Intrinsic::wasm_catch: {
  SDValue SymNode = getCppExceptionSymNode(Op, 2, DAG);
  return DAG.getNode(WebAssemblyISD::CATCH, DL,
                     {
                         MVT::i32,  // outchain type
                         MVT::Other // return value
                     },
                     {
                         Op.getOperand(0), // inchain
                         SymNode           // exception symbol
                     });
}

case Intrinsic::wasm_shuffle: {
  // Drop in-chain and replace undefs, but otherwise pass through unchanged
  SDValue Ops[18];
  size_t OpIdx = 0;
  Ops[OpIdx++] = Op.getOperand(1);
  Ops[OpIdx++] = Op.getOperand(2);
  while (OpIdx < 18) {
    const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
    if (MaskIdx.isUndef() ||
        cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) {
      Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32);
    } else {
      Ops[OpIdx++] = MaskIdx;
    }
  }
  return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
}
}
1732}

1734SDValue
1735WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
                                                SelectionDAG &DAG) const {
SDLoc DL(Op);
// If sign extension operations are disabled, allow sext_inreg only if operand
// is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
// extension operations, but allowing sext_inreg in this context lets us have
// simple patterns to select extract_lane_s instructions. Expanding sext_inreg
// everywhere would be simpler in this file, but would necessitate large and
// brittle patterns to undo the expansion and select extract_lane_s
// instructions.
assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())(static_cast <bool> (!Subtarget->hasSignExt() &&
 Subtarget->hasSIMD128()) ? void (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1745, __extension__ __PRETTY_FUNCTION__));
if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
  return SDValue();

const SDValue &Extract = Op.getOperand(0);
MVT VecT = Extract.getOperand(0).getSimpleValueType();
if (VecT.getVectorElementType().getSizeInBits() > 32)
  return SDValue();
MVT ExtractedLaneT =
    cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
MVT ExtractedVecT =
    MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
if (ExtractedVecT == VecT)
  return Op;

// Bitcast vector to appropriate type to ensure ISel pattern coverage
const SDNode *Index = Extract.getOperand(1).getNode();
if (!isa<ConstantSDNode>(Index))
  return SDValue();
unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue();
unsigned Scale =
    ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail
 ("Scale > 1", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1767, __extension__ __PRETTY_FUNCTION__));
SDValue NewIndex =
    DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
SDValue NewExtract = DAG.getNode(
    ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
    DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
                   Op.getOperand(1));
1775}

1777SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                                   SelectionDAG &DAG) const {
SDLoc DL(Op);
const EVT VecT = Op.getValueType();
const EVT LaneT = Op.getOperand(0).getValueType();
const size_t Lanes = Op.getNumOperands();
bool CanSwizzle = VecT == MVT::v16i8;

// BUILD_VECTORs are lowered to the instruction that initializes the highest
// possible number of lanes at once followed by a sequence of replace_lane
// instructions to individually initialize any remaining lanes.

// TODO: Tune this. For example, lanewise swizzling is very expensive, so
// swizzled lanes should be given greater weight.

// TODO: Investigate looping rather than always extracting/replacing specific
// lanes to fill gaps.

auto IsConstant = [](const SDValue &V) {
  return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
};

// Returns the source vector and index vector pair if they exist. Checks for:
//   (extract_vector_elt
//     $src,
//     (sign_extend_inreg (extract_vector_elt $indices, $i))
//   )
auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
  auto Bail = std::make_pair(SDValue(), SDValue());
  if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return Bail;
  const SDValue &SwizzleSrc = Lane->getOperand(0);
  const SDValue &IndexExt = Lane->getOperand(1);
  if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
    return Bail;
  const SDValue &Index = IndexExt->getOperand(0);
  if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return Bail;
  const SDValue &SwizzleIndices = Index->getOperand(0);
  if (SwizzleSrc.getValueType() != MVT::v16i8 ||
      SwizzleIndices.getValueType() != MVT::v16i8 ||
      Index->getOperand(1)->getOpcode() != ISD::Constant ||
      Index->getConstantOperandVal(1) != I)
    return Bail;
  return std::make_pair(SwizzleSrc, SwizzleIndices);
};

// If the lane is extracted from another vector at a constant index, return
// that vector. The source vector must not have more lanes than the dest
// because the shufflevector indices are in terms of the destination lanes and
// would not be able to address the smaller individual source lanes.
auto GetShuffleSrc = [&](const SDValue &Lane) {
  if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return SDValue();
  if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
    return SDValue();
  if (Lane->getOperand(0).getValueType().getVectorNumElements() >
      VecT.getVectorNumElements())
    return SDValue();
  return Lane->getOperand(0);
};

using ValueEntry = std::pair<SDValue, size_t>;
SmallVector<ValueEntry, 16> SplatValueCounts;

using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
SmallVector<SwizzleEntry, 16> SwizzleCounts;

using ShuffleEntry = std::pair<SDValue, size_t>;
SmallVector<ShuffleEntry, 16> ShuffleCounts;

auto AddCount = [](auto &Counts, const auto &Val) {
  auto CountIt =
      llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
  if (CountIt == Counts.end()) {
    Counts.emplace_back(Val, 1);
  } else {
    CountIt->second++;
  }
};

auto GetMostCommon = [](auto &Counts) {
  auto CommonIt =
      std::max_element(Counts.begin(), Counts.end(),
                       [](auto A, auto B) { return A.second < B.second; });
  assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")(static_cast <bool> (CommonIt != Counts.end() &&
 "Unexpected all-undef build_vector") ? void (0) : __assert_fail
 ("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1862, __extension__ __PRETTY_FUNCTION__));
  return *CommonIt;
};

size_t NumConstantLanes = 0;

// Count eligible lanes for each type of vector creation op
for (size_t I = 0; I2.1
'I' is < 'Lanes'
1
'I' is < 'Lanes'
 < Lanes; ++I) {
3
←
Loop condition is true.  Entering loop body→
9
←
Assuming 'I' is >= 'Lanes'→
10
←
Loop condition is false. Execution continues on line 1887→
  const SDValue &Lane = Op->getOperand(I);
  if (Lane.isUndef())
4
←
Taking false branch→
    continue;

  AddCount(SplatValueCounts, Lane);

  if (IsConstant(Lane))
5
←
Taking false branch→
    NumConstantLanes++;
  if (auto ShuffleSrc = GetShuffleSrc(Lane))
6
←
Taking false branch→
    AddCount(ShuffleCounts, ShuffleSrc);
  if (CanSwizzle6.1
'CanSwizzle' is true
1
'CanSwizzle' is true
) {
7
←
Taking true branch→
    auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
    if (SwizzleSrcs.first)
8
←
Taking false branch→
      AddCount(SwizzleCounts, SwizzleSrcs);
  }
}

SDValue SplatValue;
size_t NumSplatLanes;
std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);

SDValue SwizzleSrc;
SDValue SwizzleIndices;
size_t NumSwizzleLanes = 0;
if (SwizzleCounts.size())
11
←
Assuming the condition is true→
12
←
Taking true branch→
  std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
                        NumSwizzleLanes) = GetMostCommon(SwizzleCounts);

// Shuffles can draw from up to two vectors, so find the two most common
// sources.
SDValue ShuffleSrc1, ShuffleSrc2;
size_t NumShuffleLanes = 0;
if (ShuffleCounts.size()) {
13
←
Assuming the condition is false→
14
←
Taking false branch→
  std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
  ShuffleCounts.erase(std::remove_if(ShuffleCounts.begin(),
                                     ShuffleCounts.end(),
                                     [&](const auto &Pair) {
                                       return Pair.first == ShuffleSrc1;
                                     }),
                      ShuffleCounts.end());
}
if (ShuffleCounts.size()) {
15
←
Taking false branch→
  size_t AdditionalShuffleLanes;
  std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
      GetMostCommon(ShuffleCounts);
  NumShuffleLanes += AdditionalShuffleLanes;
}

// Predicate returning true if the lane is properly initialized by the
// original instruction
std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
SDValue Result;
// Prefer swizzles over shuffles over vector consts over splats
if (NumSwizzleLanes15.1
'NumSwizzleLanes' is >= 'NumShuffleLanes'
1
'NumSwizzleLanes' is >= 'NumShuffleLanes'
 >= NumShuffleLanes &&
17
←
Taking false branch→
    NumSwizzleLanes15.2
'NumSwizzleLanes' is >= 'NumConstantLanes'
2
'NumSwizzleLanes' is >= 'NumConstantLanes'
 >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
16
←
Assuming 'NumSwizzleLanes' is < 'NumSplatLanes'→
  Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
                       SwizzleIndices);
  auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
  IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
    return Swizzled == GetSwizzleSrcs(I, Lane);
  };
} else if (NumShuffleLanes17.1
'NumShuffleLanes' is >= 'NumConstantLanes'
1
'NumShuffleLanes' is >= 'NumConstantLanes'
 >= NumConstantLanes &&
19
←
Taking true branch→
           NumShuffleLanes >= NumSplatLanes) {
18
←
Assuming 'NumShuffleLanes' is >= 'NumSplatLanes'→
  size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
  size_t DestLaneCount = VecT.getVectorNumElements();
  size_t Scale1 = 1;
  size_t Scale2 = 1;
  SDValue Src1 = ShuffleSrc1;
20
←
Null pointer value stored to 'Src1.Node'→
  SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
21
←
'?' condition is false→
  if (Src1.getValueType() != VecT) {
22
←
Calling 'SDValue::getValueType'→
    size_t LaneSize =
        Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
    assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
 (0) : __assert_fail ("LaneSize > DestLaneSize", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1942, __extension__ __PRETTY_FUNCTION__));
    Scale1 = LaneSize / DestLaneSize;
    Src1 = DAG.getBitcast(VecT, Src1);
  }
  if (Src2.getValueType() != VecT) {
    size_t LaneSize =
        Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
    assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
 (0) : __assert_fail ("LaneSize > DestLaneSize", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1949, __extension__ __PRETTY_FUNCTION__));
    Scale2 = LaneSize / DestLaneSize;
    Src2 = DAG.getBitcast(VecT, Src2);
  }

  int Mask[16];
  assert(DestLaneCount <= 16)(static_cast <bool> (DestLaneCount <= 16) ? void (0)
 : __assert_fail ("DestLaneCount <= 16", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1955, __extension__ __PRETTY_FUNCTION__));
  for (size_t I = 0; I < DestLaneCount; ++I) {
    const SDValue &Lane = Op->getOperand(I);
    SDValue Src = GetShuffleSrc(Lane);
    if (Src == ShuffleSrc1) {
      Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
    } else if (Src && Src == ShuffleSrc2) {
      Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
    } else {
      Mask[I] = -1;
    }
  }
  ArrayRef<int> MaskRef(Mask, DestLaneCount);
  Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
  IsLaneConstructed = [&](size_t, const SDValue &Lane) {
    auto Src = GetShuffleSrc(Lane);
    return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
  };
} else if (NumConstantLanes >= NumSplatLanes) {
  SmallVector<SDValue, 16> ConstLanes;
  for (const SDValue &Lane : Op->op_values()) {
    if (IsConstant(Lane)) {
      ConstLanes.push_back(Lane);
    } else if (LaneT.isFloatingPoint()) {
      ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
    } else {
      ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
    }
  }
  Result = DAG.getBuildVector(VecT, DL, ConstLanes);
  IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
    return IsConstant(Lane);
  };
} else {
  // Use a splat, but possibly a load_splat
  LoadSDNode *SplattedLoad;
  if ((SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
      SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
    Result = DAG.getMemIntrinsicNode(
        WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT),
        {SplattedLoad->getChain(), SplattedLoad->getBasePtr(),
         SplattedLoad->getOffset()},
        SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand());
  } else {
    Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
  }
  IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
    return Lane == SplatValue;
  };
}

assert(Result)(static_cast <bool> (Result) ? void (0) : __assert_fail
 ("Result", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2006, __extension__ __PRETTY_FUNCTION__));
assert(IsLaneConstructed)(static_cast <bool> (IsLaneConstructed) ? void (0) : __assert_fail
 ("IsLaneConstructed", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2007, __extension__ __PRETTY_FUNCTION__));

// Add replace_lane instructions for any unhandled values
for (size_t I = 0; I < Lanes; ++I) {
  const SDValue &Lane = Op->getOperand(I);
  if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
    Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
                         DAG.getConstant(I, DL, MVT::i32));
}

return Result;
2018}

2020SDValue
2021WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
                                             SelectionDAG &DAG) const {
SDLoc DL(Op);
ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
MVT VecType = Op.getOperand(0).getSimpleValueType();
assert(VecType.is128BitVector() && "Unexpected shuffle vector type")(static_cast <bool> (VecType.is128BitVector() &&
 "Unexpected shuffle vector type") ? void (0) : __assert_fail
 ("VecType.is128BitVector() && \"Unexpected shuffle vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2026, __extension__ __PRETTY_FUNCTION__));
size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;

// Space for two vector args and sixteen mask indices
SDValue Ops[18];
size_t OpIdx = 0;
Ops[OpIdx++] = Op.getOperand(0);
Ops[OpIdx++] = Op.getOperand(1);

// Expand mask indices to byte indices and materialize them as operands
for (int M : Mask) {
  for (size_t J = 0; J < LaneBytes; ++J) {
    // Lower undefs (represented by -1 in mask) to zero
    uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
    Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
  }
}

return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2045}

2047SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);
// The legalizer does not know how to expand the unsupported comparison modes
// of i64x2 vectors, so we manually unroll them here.
assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64)(static_cast <bool> (Op->getOperand(0)->getSimpleValueType
(0) == MVT::v2i64) ? void (0) : __assert_fail ("Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2052, __extension__ __PRETTY_FUNCTION__));
SmallVector<SDValue, 2> LHS, RHS;
DAG.ExtractVectorElements(Op->getOperand(0), LHS);
DAG.ExtractVectorElements(Op->getOperand(1), RHS);
const SDValue &CC = Op->getOperand(2);
auto MakeLane = [&](unsigned I) {
  return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
                     DAG.getConstant(uint64_t(-1), DL, MVT::i64),
                     DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
};
return DAG.getBuildVector(Op->getValueType(0), DL,
                          {MakeLane(0), MakeLane(1)});
2064}

2066SDValue
2067WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
                                                  SelectionDAG &DAG) const {
// Allow constant lane indices, expand variable lane indices
SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
  return Op;
else
  // Perform default expansion
  return SDValue();
2076}

2078static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
EVT LaneT = Op.getSimpleValueType().getVectorElementType();
// 32-bit and 64-bit unrolled shifts will have proper semantics
if (LaneT.bitsGE(MVT::i32))
  return DAG.UnrollVectorOp(Op.getNode());
// Otherwise mask the shift value to get proper semantics from 32-bit shift
SDLoc DL(Op);
size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
unsigned ShiftOpcode = Op.getOpcode();
SmallVector<SDValue, 16> ShiftedElements;
DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
SmallVector<SDValue, 16> ShiftElements;
DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
SmallVector<SDValue, 16> UnrolledOps;
for (size_t i = 0; i < NumLanes; ++i) {
  SDValue MaskedShiftValue =
      DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
  SDValue ShiftedValue = ShiftedElements[i];
  if (ShiftOpcode == ISD::SRA)
    ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
                               ShiftedValue, DAG.getValueType(LaneT));
  UnrolledOps.push_back(
      DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
}
return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2104}

2106SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
                                            SelectionDAG &DAG) const {
SDLoc DL(Op);

// Only manually lower vector shifts
assert(Op.getSimpleValueType().isVector())(static_cast <bool> (Op.getSimpleValueType().isVector()
) ? void (0) : __assert_fail ("Op.getSimpleValueType().isVector()"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2111, __extension__ __PRETTY_FUNCTION__));

auto ShiftVal = DAG.getSplatValue(Op.getOperand(1));
if (!ShiftVal)
  return unrollVectorShift(Op, DAG);

// Use anyext because none of the high bits can affect the shift
ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);

unsigned Opcode;
switch (Op.getOpcode()) {
case ISD::SHL:
  Opcode = WebAssemblyISD::VEC_SHL;
  break;
case ISD::SRA:
  Opcode = WebAssemblyISD::VEC_SHR_S;
  break;
case ISD::SRL:
  Opcode = WebAssemblyISD::VEC_SHR_U;
  break;
default:
  llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2132);
}

return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2136}

2138SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
                                                    SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT ResT = Op.getValueType();
EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();

if ((ResT == MVT::i32 || ResT == MVT::i64) &&
    (SatVT == MVT::i32 || SatVT == MVT::i64))
  return Op;

if (ResT == MVT::v4i32 && SatVT == MVT::i32)
  return Op;

return SDValue();
2152}

2154//===----------------------------------------------------------------------===//
2155//   Custom DAG combine hooks
2156//===----------------------------------------------------------------------===//
2157static SDValue
2158performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;
auto Shuffle = cast<ShuffleVectorSDNode>(N);

// Hoist vector bitcasts that don't change the number of lanes out of unary
// shuffles, where they are less likely to get in the way of other combines.
// (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
//  (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
SDValue Bitcast = N->getOperand(0);
if (Bitcast.getOpcode() != ISD::BITCAST)
  return SDValue();
if (!N->getOperand(1).isUndef())
  return SDValue();
SDValue CastOp = Bitcast.getOperand(0);
MVT SrcType = CastOp.getSimpleValueType();
MVT DstType = Bitcast.getSimpleValueType();
if (!SrcType.is128BitVector() ||
    SrcType.getVectorNumElements() != DstType.getVectorNumElements())
  return SDValue();
SDValue NewShuffle = DAG.getVectorShuffle(
    SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
return DAG.getBitcast(DstType, NewShuffle);
2180}

2182static SDValue
2183performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;
assert(N->getOpcode() == ISD::SIGN_EXTEND ||(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
 || N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
 ("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2186, __extension__ __PRETTY_FUNCTION__))
       N->getOpcode() == ISD::ZERO_EXTEND)(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
 || N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
 ("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2186, __extension__ __PRETTY_FUNCTION__));

// Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
// possible before the extract_subvector can be expanded.
auto Extract = N->getOperand(0);
if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
  return SDValue();
auto Source = Extract.getOperand(0);
auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
if (IndexNode == nullptr)
  return SDValue();
auto Index = IndexNode->getZExtValue();

// Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
// extracted subvector is the low or high half of its source.
EVT ResVT = N->getValueType(0);
if (ResVT == MVT::v8i16) {
  if (Extract.getValueType() != MVT::v8i8 ||
      Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
    return SDValue();
} else if (ResVT == MVT::v4i32) {
  if (Extract.getValueType() != MVT::v4i16 ||
      Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
    return SDValue();
} else if (ResVT == MVT::v2i64) {
  if (Extract.getValueType() != MVT::v2i32 ||
      Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
    return SDValue();
} else {
  return SDValue();
}

bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
bool IsLow = Index == 0;

unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
                              : WebAssemblyISD::EXTEND_HIGH_S)
                     : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
                              : WebAssemblyISD::EXTEND_HIGH_U);

return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2227}

2229static SDValue
2230performVectorConvertLowCombine(SDNode *N,
                             TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;

EVT ResVT = N->getValueType(0);
if (ResVT != MVT::v2f64)
  return SDValue();

auto GetWasmConversionOp = [](unsigned Op) {
  switch (Op) {
  case ISD::SINT_TO_FP:
    return WebAssemblyISD::CONVERT_LOW_S;
  case ISD::UINT_TO_FP:
    return WebAssemblyISD::CONVERT_LOW_U;
  case ISD::FP_EXTEND:
    return WebAssemblyISD::PROMOTE_LOW;
  }
  llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2247);
};

if (N->getOpcode() == ISD::EXTRACT_SUBVECTOR) {
  // Combine this:
  //
  //   (v2f64 (extract_subvector
  //     (v4f64 ({s,u}int_to_fp (v4i32 $x))), 0))
  //
  // into (f64x2.convert_low_i32x4_{s,u} $x).
  //
  // Or this:
  //
  //  (v2f64 (extract_subvector
  //    (v4f64 (fp_extend (v4f32 $x))), 0))
  //
  // into (f64x2.promote_low_f32x4 $x).
  auto Conversion = N->getOperand(0);
  auto ConversionOp = Conversion.getOpcode();
  MVT ExpectedSourceType;
  switch (ConversionOp) {
  case ISD::SINT_TO_FP:
  case ISD::UINT_TO_FP:
    ExpectedSourceType = MVT::v4i32;
    break;
  case ISD::FP_EXTEND:
    ExpectedSourceType = MVT::v4f32;
    break;
  default:
    return SDValue();
  }

  if (Conversion.getValueType() != MVT::v4f64)
    return SDValue();

  auto Source = Conversion.getOperand(0);
  if (Source.getValueType() != ExpectedSourceType)
    return SDValue();

  auto IndexNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (IndexNode == nullptr || IndexNode->getZExtValue() != 0)
    return SDValue();

  auto Op = GetWasmConversionOp(ConversionOp);
  return DAG.getNode(Op, SDLoc(N), ResVT, Source);
}

// Combine this:
//
//   (v2f64 ({s,u}int_to_fp
//     (v2i32 (extract_subvector (v4i32 $x), 0))))
//
// into (f64x2.convert_low_i32x4_{s,u} $x).
//
// Or this:
//
//   (v2f64 (fp_extend
//     (v2f32 (extract_subvector (v4f32 $x), 0))))
//
// into (f64x2.promote_low_f32x4 $x).
auto ConversionOp = N->getOpcode();
MVT ExpectedExtractType;
MVT ExpectedSourceType;
switch (ConversionOp) {
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP:
  ExpectedExtractType = MVT::v2i32;
  ExpectedSourceType = MVT::v4i32;
  break;
case ISD::FP_EXTEND:
  ExpectedExtractType = MVT::v2f32;
  ExpectedSourceType = MVT::v4f32;
  break;
default:
  llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2321);
}

auto Extract = N->getOperand(0);
if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
  return SDValue();

if (Extract.getValueType() != ExpectedExtractType)
  return SDValue();

auto Source = Extract.getOperand(0);
if (Source.getValueType() != ExpectedSourceType)
  return SDValue();

auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
if (IndexNode == nullptr || IndexNode->getZExtValue() != 0)
  return SDValue();

unsigned Op = GetWasmConversionOp(ConversionOp);
return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2341}

2343static SDValue
2344performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
auto &DAG = DCI.DAG;

auto GetWasmConversionOp = [](unsigned Op) {
  switch (Op) {
  case ISD::FP_TO_SINT_SAT:
    return WebAssemblyISD::TRUNC_SAT_ZERO_S;
  case ISD::FP_TO_UINT_SAT:
    return WebAssemblyISD::TRUNC_SAT_ZERO_U;
  case ISD::FP_ROUND:
    return WebAssemblyISD::DEMOTE_ZERO;
  }
  llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2356);
};

auto IsZeroSplat = [](SDValue SplatVal) {
  auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
  APInt SplatValue, SplatUndef;
  unsigned SplatBitSize;
  bool HasAnyUndefs;
  return Splat &&
         Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
                                HasAnyUndefs) &&
         SplatValue == 0;
};

if (N->getOpcode() == ISD::CONCAT_VECTORS) {
  // Combine this:
  //
  //   (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
  //
  // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
  //
  // Or this:
  //
  //   (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
  //
  // into (f32x4.demote_zero_f64x2 $x).
  EVT ResVT;
  EVT ExpectedConversionType;
  auto Conversion = N->getOperand(0);
  auto ConversionOp = Conversion.getOpcode();
  switch (ConversionOp) {
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
    ResVT = MVT::v4i32;
    ExpectedConversionType = MVT::v2i32;
    break;
  case ISD::FP_ROUND:
    ResVT = MVT::v4f32;
    ExpectedConversionType = MVT::v2f32;
    break;
  default:
    return SDValue();
  }

  if (N->getValueType(0) != ResVT)
    return SDValue();

  if (Conversion.getValueType() != ExpectedConversionType)
    return SDValue();

  auto Source = Conversion.getOperand(0);
  if (Source.getValueType() != MVT::v2f64)
    return SDValue();

  if (!IsZeroSplat(N->getOperand(1)) ||
      N->getOperand(1).getValueType() != ExpectedConversionType)
    return SDValue();

  unsigned Op = GetWasmConversionOp(ConversionOp);
  return DAG.getNode(Op, SDLoc(N), ResVT, Source);
}

// Combine this:
//
//   (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
//
// into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
//
// Or this:
//
//   (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
//
// into (f32x4.demote_zero_f64x2 $x).
EVT ResVT;
auto ConversionOp = N->getOpcode();
switch (ConversionOp) {
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
  ResVT = MVT::v4i32;
  break;
case ISD::FP_ROUND:
  ResVT = MVT::v4f32;
  break;
default:
  llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2440);
}

if (N->getValueType(0) != ResVT)
  return SDValue();

auto Concat = N->getOperand(0);
if (Concat.getValueType() != MVT::v4f64)
  return SDValue();

auto Source = Concat.getOperand(0);
if (Source.getValueType() != MVT::v2f64)
  return SDValue();

if (!IsZeroSplat(Concat.getOperand(1)) ||
    Concat.getOperand(1).getValueType() != MVT::v2f64)
  return SDValue();

unsigned Op = GetWasmConversionOp(ConversionOp);
return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2460}

2462SDValue
2463WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
                                           DAGCombinerInfo &DCI) const {
switch (N->getOpcode()) {
default:
  return SDValue();
case ISD::VECTOR_SHUFFLE:
  return performVECTOR_SHUFFLECombine(N, DCI);
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
  return performVectorExtendCombine(N, DCI);
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP:
case ISD::FP_EXTEND:
case ISD::EXTRACT_SUBVECTOR:
  return performVectorConvertLowCombine(N, DCI);
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
case ISD::FP_ROUND:
case ISD::CONCAT_VECTORS:
  return performVectorTruncZeroCombine(N, DCI);
}
2484}

←

/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>

57namespace llvm {

59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;

72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG.  Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
82};

84namespace ISD {

/// Node predicates

88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);

109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);

113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);

117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);

125} // end namespace ISD

127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation.  Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node.  This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
unsigned ResNo = 0;     // Which return value of the node we are using.

144public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
230};

232template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
253};

255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
}
263};
264template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
270};

272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

287public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

327private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
356};

358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
366};

368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

391public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
451};

453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
/// The operation that this node performs.
int16_t NodeType;

460protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with NodeType.

465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif

475BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
557END_TWO_BYTE_PACK()
558#undef BEGIN_TWO_BYTE_PACK
559#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

571private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

608public:
/// Unique and persistent id per SDNode in the DAG.
/// Used for debug printing.
uint16_t PersistentId;

//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned short)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
664#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
666#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 679, __extension__ __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using iterator_category = std::forward_iterator_tag;
  using value_type = SDUse;
  using difference_type = std::ptrdiff_t;
  using pointer = value_type *;
  using reference = value_type &;

  use_iterator() = default;
  use_iterator(const use_iterator &I) : Op(I.Op) {}

  bool operator==(const use_iterator &x) const {
    return Op == x.Op;
  }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 748, __extension__ __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 759, __extension__ __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __extension__ __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 897, __extension__ __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 959, __extension__ __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1046protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
 "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1060, __extension__ __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1062, __extension__ __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1067};

1069/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1070/// into SDNode creation functions.
1071/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1072/// from the original Instruction, and IROrder is the ordinal position of
1073/// the instruction.
1074/// When an SDNode is created after the DAG is being built, both DebugLoc and
1075/// the IROrder are propagated from the original SDNode.
1076/// So SDLoc class provides two constructors besides the default one, one to
1077/// be used by the DAGBuilder, the other to be used by others.
1078class SDLoc {
1079private:
DebugLoc DL;
int IROrder = 0;

1083public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1088, __extension__ __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1095};

1097// Define inline functions from the SDValue class.

1099inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1105, __extension__ __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1106, __extension__ __PRETTY_FUNCTION__));
1107}

1109inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
1111}

1113inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
23
←
Called C++ object pointer is null
1115}

1117inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1119}

1121inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1123}

1125inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1127}

1129inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1131}

1133inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1135}

1137inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1139}

1141inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1143}

1145inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1147}

1149inline bool SDValue::isUndef() const {
return Node->isUndef();
1151}

1153inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1155}

1157inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
1159}

1161inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1163}

1165inline void SDValue::dump() const {
return Node->dump();
1167}

1169inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1171}

1173inline void SDValue::dumpr() const {
return Node->dumpr();
1175}

1177inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1179}

1181// Define inline functions from the SDUse class.

1183inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode()) V.getNode()->addUse(*this);
1187}

1189inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V.getNode()->addUse(*this);
1192}

1194inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1198}

1200/// This class is used to form a handle around another node that
1201/// is persistent and is updated across invocations of replaceAllUsesWith on its
1202/// operand.  This node should be directly created by end-users and not added to
1203/// the AllNodes list.
1204class HandleSDNode : public SDNode {
SDUse Op;

1207public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1226};

1228class AddrSpaceCastSDNode : public SDNode {
1229private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1233public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1243};

1245/// This is an abstract virtual class for memory operations.
1246class MemSDNode : public SDNode {
1247private:
// VT of in-memory value.
EVT MemoryVT;

1251protected:
/// Memory reference information.
MachineMemOperand *MMO;

1255public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }
// FIXME: Remove once transition to getAlign is over.
unsigned getAlignment() const { return MMO->getAlign().value(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getSuccessOrdering() const {
  return MMO->getSuccessOrdering();
}

/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation.  (For operations
/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::MSTORE:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  switch (N->getOpcode()) {
  case ISD::LOAD:
  case ISD::STORE:
  case ISD::PREFETCH:
  case ISD::ATOMIC_CMP_SWAP:
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  case ISD::ATOMIC_SWAP:
  case ISD::ATOMIC_LOAD_ADD:
  case ISD::ATOMIC_LOAD_SUB:
  case ISD::ATOMIC_LOAD_AND:
  case ISD::ATOMIC_LOAD_CLR:
  case ISD::ATOMIC_LOAD_OR:
  case ISD::ATOMIC_LOAD_XOR:
  case ISD::ATOMIC_LOAD_NAND:
  case ISD::ATOMIC_LOAD_MIN:
  case ISD::ATOMIC_LOAD_MAX:
  case ISD::ATOMIC_LOAD_UMIN:
  case ISD::ATOMIC_LOAD_UMAX:
  case ISD::ATOMIC_LOAD_FADD:
  case ISD::ATOMIC_LOAD_FSUB:
  case ISD::ATOMIC_LOAD:
  case ISD::ATOMIC_STORE:
  case ISD::MLOAD:
  case ISD::MSTORE:
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return true;
  default:
    return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
  }
}
1401};

1403/// This is an SDNode representing atomic operations.
1404class AtomicSDNode : public MemSDNode {
1405public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1410, __extension__ __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1427, __extension__ __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1452};

1454/// This SDNode is used for target intrinsics that touch
1455/// memory and need an associated MachineMemOperand. Its opcode may be
1456/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1457/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1458class MemIntrinsicSDNode : public MemSDNode {
1459public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1474};

1476/// This SDNode is used to implement the code generator
1477/// support for the llvm IR shufflevector instruction.  It combines elements
1478/// from two input vectors into a new input vector, with the selection and
1479/// ordering of elements determined by an array of integers, referred to as
1480/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1481/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1482/// An index of -1 is treated as undef, such that the code generator may put
1483/// any value in the corresponding element of the result.
1484class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1489protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1495public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return makeArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
 ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1502, __extension__ __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1509, __extension__ __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1540};

1542class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1554public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isNullValue() const { return Value->isZero(); }
bool isAllOnesValue() const { return Value->isMinusOne(); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1575};

1577uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1579}

1581const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1583}

1585class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1595public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1630};

1632/// Returns true if \p V is a constant integer zero.
1633bool isNullConstant(SDValue V);

1635/// Returns true if \p V is an FP constant with a value of positive zero.
1636bool isNullFPConstant(SDValue V);

1638/// Returns true if \p V is an integer constant with all bits set.
1639bool isAllOnesConstant(SDValue V);

1641/// Returns true if \p V is a constant integer one.
1642bool isOneConstant(SDValue V);

1644/// Return the non-bitcasted source operand of \p V if it exists.
1645/// If \p V is not a bitcasted value, it is returned as-is.
1646SDValue peekThroughBitcasts(SDValue V);

1648/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1649/// If \p V is not a bitcasted one-use value, it is returned as-is.
1650SDValue peekThroughOneUseBitcasts(SDValue V);

1652/// Return the non-extracted vector source operand of \p V if it exists.
1653/// If \p V is not an extracted subvector, it is returned as-is.
1654SDValue peekThroughExtractSubvectors(SDValue V);

1656/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1657/// constant is canonicalized to be operand 1.
1658bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1660/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1661ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1664/// Returns the SDNode if it is a demanded constant splat BuildVector or
1665/// constant int.
1666ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1670/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1671ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1673/// Returns the SDNode if it is a demanded constant splat BuildVector or
1674/// constant float.
1675ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1678/// Return true if the value is a constant 0 integer or a splatted vector of
1679/// a constant 0 integer (with no undefs by default).
1680/// Build vector implicit truncation is not an issue for null values.
1681bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1683/// Return true if the value is a constant 1 integer or a splatted vector of a
1684/// constant 1 integer (with no undefs).
1685/// Does not permit build vector implicit truncation.
1686bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1688/// Return true if the value is a constant -1 integer or a splatted vector of a
1689/// constant -1 integer (with no undefs).
1690/// Does not permit build vector implicit truncation.
1691bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1693/// Return true if \p V is either a integer or FP constant.
1694inline bool isIntOrFPConstant(SDValue V) {
return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1696}

1698class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1709public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1722};

1724class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1734public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1741};

1743/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1744/// the offet and size that are started/ended in the underlying FrameIndex.
1745class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1753public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1760, __extension__ __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1764, __extension__ __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1773};

1775/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1776/// the index of the basic block being probed. A pseudo probe serves as a place
1777/// holder and will be removed at the end of compilation. It does not have any
1778/// operand because we do not want the instruction selection to deal with any.
1779class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1790public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1799};

1801class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1812public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1820};

1822class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1838, __extension__ __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1847, __extension__ __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1852public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1858, __extension__ __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1863, __extension__ __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1882};

1884/// Completely target-dependent object reference.
1885class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1892public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1904};

1906class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1918public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1924};

1926/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1927class BuildVectorSDNode : public SDNode {
1928public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

bool isConstant() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2037};

2039/// An SDNode that holds an arbitrary LLVM IR Value. This is
2040/// used when the SelectionDAG needs to make a simple reference to something
2041/// in the LLVM IR representation.
2042///
2043class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2052public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2059};

2061class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2070public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2076};

2078class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2086public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2092};

2094class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2104public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2110};

2112class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2124public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2133};

2135class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
 "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2142, __extension__ __PRETTY_FUNCTION__));
}

2145public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2152};

2154class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2165public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2173};

2175class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2183public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2189};

2191class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2200public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2206};

2208/// This class is used to represent EVT's, which are used
2209/// to parameterize some operations.
2210class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2219public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2225};

2227/// Base class for LoadSDNode and StoreSDNode
2228class LSBaseSDNode : public MemSDNode {
2229public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2235, __extension__ __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2258};

2260/// This class is used to represent ISD::LOAD nodes.
2261class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2269, __extension__ __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2270, __extension__ __PRETTY_FUNCTION__));
}

2273public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2286};

2288/// This class is used to represent ISD::STORE nodes.
2289class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2297, __extension__ __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2298, __extension__ __PRETTY_FUNCTION__));
}

2301public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2317};

2319/// This base class is used to represent MLOAD and MSTORE nodes
2320class MaskedLoadStoreSDNode : public MemSDNode {
2321public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2330, __extension__ __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2359};

2361/// This class is used to represent an MLOAD node
2362class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2363public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2388};

2390/// This class is used to represent an MSTORE node
2391class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2392public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2422};

2424/// This is a base class used to represent
2425/// MGATHER and MSCATTER nodes
2426///
2427class MaskedGatherScatterSDNode : public MemSDNode {
2428public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2436, __extension__ __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
void setIndexType(ISD::MemIndexType IndexType) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2468};

2470/// This class is used to represent an MGATHER node
2471///
2472class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2473public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2493};

2495/// This class is used to represent an MSCATTER node
2496///
2497class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2498public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2519};

2521/// An SDNode that represents everything that will be needed
2522/// to construct a MachineInstr. These nodes are created during the
2523/// instruction selection proper phase.
2524///
2525/// Note that the only supported way to set the `memoperands` is by calling the
2526/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2527/// inside the DAG rather than in the node.
2528class MachineSDNode : public SDNode {
2529private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2557public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2583};

2585/// An SDNode that records if a register contains a value that is guaranteed to
2586/// be aligned accordingly.
2587class AssertAlignSDNode : public SDNode {
Align Alignment;

2590public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2599};

2601class SDNodeIterator {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2607public:
using iterator_category = std::forward_iterator_tag;
using value_type = SDNode;
using difference_type = std::ptrdiff_t;
using pointer = value_type *;
using reference = value_type &;

bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2633, __extension__ __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2633, __extension__ __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
2644};

2646template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
2659};

2661/// A representation of the largest SDNode, for use in sizeof().
2662///
2663/// This needs to be a union because the largest node differs on 32 bit systems
2664/// with 4 and 8 byte pointer alignment, respectively.
2665using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

2670/// The SDNode class with the greatest alignment requirement.
2671using MostAlignedSDNode = GlobalAddressSDNode;

2673namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

2751} // end namespace ISD

2753} // end namespace llvm

2755#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H