/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Bug Summary

File:	build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1170, column 10 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name AMDGPUISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-15/lib/clang/15.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/Target/AMDGPU -I include -I /build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-15/lib/clang/15.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/= -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-04-20-140412-16051-1 -x c++ /build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

→

1//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This is the parent TargetLowering class for hardware code gen
11/// targets.
12//
13//===----------------------------------------------------------------------===//

15#include "AMDGPUISelLowering.h"
16#include "AMDGPU.h"
17#include "AMDGPUInstrInfo.h"
18#include "AMDGPUMachineFunction.h"
19#include "GCNSubtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "llvm/CodeGen/Analysis.h"
22#include "llvm/CodeGen/MachineFrameInfo.h"
23#include "llvm/IR/DiagnosticInfo.h"
24#include "llvm/IR/IntrinsicsAMDGPU.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/KnownBits.h"
27#include "llvm/Target/TargetMachine.h"

29using namespace llvm;

31#include "AMDGPUGenCallingConv.inc"

33static cl::opt<bool> AMDGPUBypassSlowDiv(
"amdgpu-bypass-slow-div",
cl::desc("Skip 64-bit divide for dynamic 32-bit values"),
cl::init(true));

38// Find a larger type to do a load / store of a vector with.
39EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
unsigned StoreSize = VT.getStoreSizeInBits();
if (StoreSize <= 32)
  return EVT::getIntegerVT(Ctx, StoreSize);

assert(StoreSize % 32 == 0 && "Store size not a multiple of 32")(static_cast <bool> (StoreSize % 32 == 0 && "Store size not a multiple of 32"
) ? void (0) : __assert_fail ("StoreSize % 32 == 0 && \"Store size not a multiple of 32\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 44, __extension__
 __PRETTY_FUNCTION__));
return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
46}

48unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
return DAG.computeKnownBits(Op).countMaxActiveBits();
50}

52unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
// In order for this to be a signed 24-bit value, bit 23, must
// be a sign bit.
return DAG.ComputeMaxSignificantBits(Op);
56}

58AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
                                         const AMDGPUSubtarget &STI)
  : TargetLowering(TM), Subtarget(&STI) {
// Lower floating point store/load to integer store/load to reduce the number
// of patterns in tablegen.
setOperationAction(ISD::LOAD, MVT::f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);

setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32);

setOperationAction(ISD::LOAD, MVT::v3f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32);

setOperationAction(ISD::LOAD, MVT::v4f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32);

setOperationAction(ISD::LOAD, MVT::v5f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32);

setOperationAction(ISD::LOAD, MVT::v6f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v6f32, MVT::v6i32);

setOperationAction(ISD::LOAD, MVT::v7f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v7f32, MVT::v7i32);

setOperationAction(ISD::LOAD, MVT::v8f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32);

setOperationAction(ISD::LOAD, MVT::v16f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32);

setOperationAction(ISD::LOAD, MVT::v32f32, Promote);
AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32);

setOperationAction(ISD::LOAD, MVT::i64, Promote);
AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32);

setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32);

setOperationAction(ISD::LOAD, MVT::f64, Promote);
AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32);

setOperationAction(ISD::LOAD, MVT::v2f64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32);

setOperationAction(ISD::LOAD, MVT::v3i64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v3i64, MVT::v6i32);

setOperationAction(ISD::LOAD, MVT::v4i64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4i64, MVT::v8i32);

setOperationAction(ISD::LOAD, MVT::v3f64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v3f64, MVT::v6i32);

setOperationAction(ISD::LOAD, MVT::v4f64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v4f64, MVT::v8i32);

setOperationAction(ISD::LOAD, MVT::v8i64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v8i64, MVT::v16i32);

setOperationAction(ISD::LOAD, MVT::v8f64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v8f64, MVT::v16i32);

setOperationAction(ISD::LOAD, MVT::v16i64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v16i64, MVT::v32i32);

setOperationAction(ISD::LOAD, MVT::v16f64, Promote);
AddPromotedToType(ISD::LOAD, MVT::v16f64, MVT::v32i32);

// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes()) {
  setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand);
  setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand);
  setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand);
}

for (MVT VT : MVT::integer_valuetypes()) {
  if (VT == MVT::i64)
    continue;

  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Legal);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Legal);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);

  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Legal);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Legal);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);

  setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Legal);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Legal);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
}

for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v3i16, Expand);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v3i16, Expand);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v3i16, Expand);
  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand);
  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand);
  setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand);
}

setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand);

setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f32, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f32, Expand);

setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v3f64, MVT::v3f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand);
setLoadExtAction(ISD::EXTLOAD, MVT::v16f64, MVT::v16f16, Expand);

setOperationAction(ISD::STORE, MVT::f32, Promote);
AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32);

setOperationAction(ISD::STORE, MVT::v2f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32);

setOperationAction(ISD::STORE, MVT::v3f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32);

setOperationAction(ISD::STORE, MVT::v4f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32);

setOperationAction(ISD::STORE, MVT::v5f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32);

setOperationAction(ISD::STORE, MVT::v6f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v6f32, MVT::v6i32);

setOperationAction(ISD::STORE, MVT::v7f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v7f32, MVT::v7i32);

setOperationAction(ISD::STORE, MVT::v8f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32);

setOperationAction(ISD::STORE, MVT::v16f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32);

setOperationAction(ISD::STORE, MVT::v32f32, Promote);
AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32);

setOperationAction(ISD::STORE, MVT::i64, Promote);
AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32);

setOperationAction(ISD::STORE, MVT::v2i64, Promote);
AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32);

setOperationAction(ISD::STORE, MVT::f64, Promote);
AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32);

setOperationAction(ISD::STORE, MVT::v2f64, Promote);
AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32);

setOperationAction(ISD::STORE, MVT::v3i64, Promote);
AddPromotedToType(ISD::STORE, MVT::v3i64, MVT::v6i32);

setOperationAction(ISD::STORE, MVT::v3f64, Promote);
AddPromotedToType(ISD::STORE, MVT::v3f64, MVT::v6i32);

setOperationAction(ISD::STORE, MVT::v4i64, Promote);
AddPromotedToType(ISD::STORE, MVT::v4i64, MVT::v8i32);

setOperationAction(ISD::STORE, MVT::v4f64, Promote);
AddPromotedToType(ISD::STORE, MVT::v4f64, MVT::v8i32);

setOperationAction(ISD::STORE, MVT::v8i64, Promote);
AddPromotedToType(ISD::STORE, MVT::v8i64, MVT::v16i32);

setOperationAction(ISD::STORE, MVT::v8f64, Promote);
AddPromotedToType(ISD::STORE, MVT::v8f64, MVT::v16i32);

setOperationAction(ISD::STORE, MVT::v16i64, Promote);
AddPromotedToType(ISD::STORE, MVT::v16i64, MVT::v32i32);

setOperationAction(ISD::STORE, MVT::v16f64, Promote);
AddPromotedToType(ISD::STORE, MVT::v16f64, MVT::v32i32);

setTruncStoreAction(MVT::i64, MVT::i1, Expand);
setTruncStoreAction(MVT::i64, MVT::i8, Expand);
setTruncStoreAction(MVT::i64, MVT::i16, Expand);
setTruncStoreAction(MVT::i64, MVT::i32, Expand);

setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand);
setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand);
setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand);
setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand);

setTruncStoreAction(MVT::f32, MVT::f16, Expand);
setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand);
setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand);
setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand);
setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand);
setTruncStoreAction(MVT::v16f32, MVT::v16f16, Expand);
setTruncStoreAction(MVT::v32f32, MVT::v32f16, Expand);

setTruncStoreAction(MVT::f64, MVT::f16, Expand);
setTruncStoreAction(MVT::f64, MVT::f32, Expand);

setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand);
setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand);

setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
setTruncStoreAction(MVT::v3f64, MVT::v3f32, Expand);
setTruncStoreAction(MVT::v3f64, MVT::v3f16, Expand);

setTruncStoreAction(MVT::v4i64, MVT::v4i32, Expand);
setTruncStoreAction(MVT::v4i64, MVT::v4i16, Expand);
setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand);
setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand);

setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand);
setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand);

setTruncStoreAction(MVT::v16f64, MVT::v16f32, Expand);
setTruncStoreAction(MVT::v16f64, MVT::v16f16, Expand);
setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand);
setTruncStoreAction(MVT::v16i64, MVT::v16i16, Expand);
setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand);
setTruncStoreAction(MVT::v16i64, MVT::v16i8, Expand);
setTruncStoreAction(MVT::v16i64, MVT::v16i1, Expand);

setOperationAction(ISD::Constant, MVT::i32, Legal);
setOperationAction(ISD::Constant, MVT::i64, Legal);
setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
setOperationAction(ISD::ConstantFP, MVT::f64, Legal);

setOperationAction(ISD::BR_JT, MVT::Other, Expand);
setOperationAction(ISD::BRIND, MVT::Other, Expand);

// This is totally unsupported, just custom lower to produce an error.
setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);

// Library functions.  These default to Expand, but we have instructions
// for them.
setOperationAction(ISD::FCEIL,  MVT::f32, Legal);
setOperationAction(ISD::FEXP2,  MVT::f32, Legal);
setOperationAction(ISD::FPOW,   MVT::f32, Legal);
setOperationAction(ISD::FLOG2,  MVT::f32, Legal);
setOperationAction(ISD::FABS,   MVT::f32, Legal);
setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
setOperationAction(ISD::FRINT,  MVT::f32, Legal);
setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);

setOperationAction(ISD::FROUND, MVT::f32, Custom);
setOperationAction(ISD::FROUND, MVT::f64, Custom);

setOperationAction(ISD::FLOG, MVT::f32, Custom);
setOperationAction(ISD::FLOG10, MVT::f32, Custom);
setOperationAction(ISD::FEXP, MVT::f32, Custom);


setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom);
setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom);

setOperationAction(ISD::FREM, MVT::f16, Custom);
setOperationAction(ISD::FREM, MVT::f32, Custom);
setOperationAction(ISD::FREM, MVT::f64, Custom);

// Expand to fneg + fadd.
setOperationAction(ISD::FSUB, MVT::f64, Expand);

setOperationAction(ISD::CONCAT_VECTORS, MVT::v3i32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v3f32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v5i32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v5f32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v6i32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v6f32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v7i32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v7f32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom);
setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f16, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i16, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f16, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i16, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v3f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v3i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v5f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v5i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v6f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v6i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v7f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v7i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v32f32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v32i32, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v3f64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v3i64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f64, Custom);
setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16i64, Custom);

setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);

const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
for (MVT VT : ScalarIntVTs) {
  // These should use [SU]DIVREM, so set them to expand
  setOperationAction(ISD::SDIV, VT, Expand);
  setOperationAction(ISD::UDIV, VT, Expand);
  setOperationAction(ISD::SREM, VT, Expand);
  setOperationAction(ISD::UREM, VT, Expand);

  // GPU does not have divrem function for signed or unsigned.
  setOperationAction(ISD::SDIVREM, VT, Custom);
  setOperationAction(ISD::UDIVREM, VT, Custom);

  // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
  setOperationAction(ISD::SMUL_LOHI, VT, Expand);
  setOperationAction(ISD::UMUL_LOHI, VT, Expand);

  setOperationAction(ISD::BSWAP, VT, Expand);
  setOperationAction(ISD::CTTZ, VT, Expand);
  setOperationAction(ISD::CTLZ, VT, Expand);

  // AMDGPU uses ADDC/SUBC/ADDE/SUBE
  setOperationAction(ISD::ADDC, VT, Legal);
  setOperationAction(ISD::SUBC, VT, Legal);
  setOperationAction(ISD::ADDE, VT, Legal);
  setOperationAction(ISD::SUBE, VT, Legal);
}

// The hardware supports 32-bit FSHR, but not FSHL.
setOperationAction(ISD::FSHR, MVT::i32, Legal);

// The hardware supports 32-bit ROTR, but not ROTL.
setOperationAction(ISD::ROTL, MVT::i32, Expand);
setOperationAction(ISD::ROTL, MVT::i64, Expand);
setOperationAction(ISD::ROTR, MVT::i64, Expand);

setOperationAction(ISD::MULHU, MVT::i16, Expand);
setOperationAction(ISD::MULHS, MVT::i16, Expand);

setOperationAction(ISD::MUL, MVT::i64, Expand);
setOperationAction(ISD::MULHU, MVT::i64, Expand);
setOperationAction(ISD::MULHS, MVT::i64, Expand);
setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);

setOperationAction(ISD::SMIN, MVT::i32, Legal);
setOperationAction(ISD::UMIN, MVT::i32, Legal);
setOperationAction(ISD::SMAX, MVT::i32, Legal);
setOperationAction(ISD::UMAX, MVT::i32, Legal);

setOperationAction(ISD::CTTZ, MVT::i64, Custom);
setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Custom);
setOperationAction(ISD::CTLZ, MVT::i64, Custom);
setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);

static const MVT::SimpleValueType VectorIntTypes[] = {
    MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32};

for (MVT VT : VectorIntTypes) {
  // Expand the following operations for the current type by default.
  setOperationAction(ISD::ADD,  VT, Expand);
  setOperationAction(ISD::AND,  VT, Expand);
  setOperationAction(ISD::FP_TO_SINT, VT, Expand);
  setOperationAction(ISD::FP_TO_UINT, VT, Expand);
  setOperationAction(ISD::MUL,  VT, Expand);
  setOperationAction(ISD::MULHU, VT, Expand);
  setOperationAction(ISD::MULHS, VT, Expand);
  setOperationAction(ISD::OR,   VT, Expand);
  setOperationAction(ISD::SHL,  VT, Expand);
  setOperationAction(ISD::SRA,  VT, Expand);
  setOperationAction(ISD::SRL,  VT, Expand);
  setOperationAction(ISD::ROTL, VT, Expand);
  setOperationAction(ISD::ROTR, VT, Expand);
  setOperationAction(ISD::SUB,  VT, Expand);
  setOperationAction(ISD::SINT_TO_FP, VT, Expand);
  setOperationAction(ISD::UINT_TO_FP, VT, Expand);
  setOperationAction(ISD::SDIV, VT, Expand);
  setOperationAction(ISD::UDIV, VT, Expand);
  setOperationAction(ISD::SREM, VT, Expand);
  setOperationAction(ISD::UREM, VT, Expand);
  setOperationAction(ISD::SMUL_LOHI, VT, Expand);
  setOperationAction(ISD::UMUL_LOHI, VT, Expand);
  setOperationAction(ISD::SDIVREM, VT, Expand);
  setOperationAction(ISD::UDIVREM, VT, Expand);
  setOperationAction(ISD::SELECT, VT, Expand);
  setOperationAction(ISD::VSELECT, VT, Expand);
  setOperationAction(ISD::SELECT_CC, VT, Expand);
  setOperationAction(ISD::XOR,  VT, Expand);
  setOperationAction(ISD::BSWAP, VT, Expand);
  setOperationAction(ISD::CTPOP, VT, Expand);
  setOperationAction(ISD::CTTZ, VT, Expand);
  setOperationAction(ISD::CTLZ, VT, Expand);
  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
  setOperationAction(ISD::SETCC, VT, Expand);
}

static const MVT::SimpleValueType FloatVectorTypes[] = {
    MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32};

for (MVT VT : FloatVectorTypes) {
  setOperationAction(ISD::FABS, VT, Expand);
  setOperationAction(ISD::FMINNUM, VT, Expand);
  setOperationAction(ISD::FMAXNUM, VT, Expand);
  setOperationAction(ISD::FADD, VT, Expand);
  setOperationAction(ISD::FCEIL, VT, Expand);
  setOperationAction(ISD::FCOS, VT, Expand);
  setOperationAction(ISD::FDIV, VT, Expand);
  setOperationAction(ISD::FEXP2, VT, Expand);
  setOperationAction(ISD::FEXP, VT, Expand);
  setOperationAction(ISD::FLOG2, VT, Expand);
  setOperationAction(ISD::FREM, VT, Expand);
  setOperationAction(ISD::FLOG, VT, Expand);
  setOperationAction(ISD::FLOG10, VT, Expand);
  setOperationAction(ISD::FPOW, VT, Expand);
  setOperationAction(ISD::FFLOOR, VT, Expand);
  setOperationAction(ISD::FTRUNC, VT, Expand);
  setOperationAction(ISD::FMUL, VT, Expand);
  setOperationAction(ISD::FMA, VT, Expand);
  setOperationAction(ISD::FRINT, VT, Expand);
  setOperationAction(ISD::FNEARBYINT, VT, Expand);
  setOperationAction(ISD::FSQRT, VT, Expand);
  setOperationAction(ISD::FSIN, VT, Expand);
  setOperationAction(ISD::FSUB, VT, Expand);
  setOperationAction(ISD::FNEG, VT, Expand);
  setOperationAction(ISD::VSELECT, VT, Expand);
  setOperationAction(ISD::SELECT_CC, VT, Expand);
  setOperationAction(ISD::FCOPYSIGN, VT, Expand);
  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
  setOperationAction(ISD::SETCC, VT, Expand);
  setOperationAction(ISD::FCANONICALIZE, VT, Expand);
}

// This causes using an unrolled select operation rather than expansion with
// bit operations. This is in general better, but the alternative using BFI
// instructions may be better if the select sources are SGPRs.
setOperationAction(ISD::SELECT, MVT::v2f32, Promote);
AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32);

setOperationAction(ISD::SELECT, MVT::v3f32, Promote);
AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32);

setOperationAction(ISD::SELECT, MVT::v4f32, Promote);
AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32);

setOperationAction(ISD::SELECT, MVT::v5f32, Promote);
AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32);

setOperationAction(ISD::SELECT, MVT::v6f32, Promote);
AddPromotedToType(ISD::SELECT, MVT::v6f32, MVT::v6i32);

setOperationAction(ISD::SELECT, MVT::v7f32, Promote);
AddPromotedToType(ISD::SELECT, MVT::v7f32, MVT::v7i32);

// There are no libcalls of any kind.
for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
  setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);

setSchedulingPreference(Sched::RegPressure);
setJumpIsExpensive(true);

// FIXME: This is only partially true. If we have to do vector compares, any
// SGPR pair can be a condition register. If we have a uniform condition, we
// are better off doing SALU operations, where there is only one SCC. For now,
// we don't have a way of knowing during instruction selection if a condition
// will be uniform and we always use vector compares. Assume we are using
// vector compares until that is fixed.
setHasMultipleConditionRegisters(true);

setMinCmpXchgSizeInBits(32);
setSupportsUnalignedAtomics(false);

PredictableSelectIsExpensive = false;

// We want to find all load dependencies for long chains of stores to enable
// merging into very wide vectors. The problem is with vectors with > 4
// elements. MergeConsecutiveStores will attempt to merge these because x8/x16
// vectors are a legal type, even though we have to split the loads
// usually. When we can more precisely specify load legality per address
// space, we should be able to make FindBetterChain/MergeConsecutiveStores
// smarter so that they can figure out what to do in 2 iterations without all
// N > 4 stores on the same chain.
GatherAllAliasesMaxDepth = 16;

// memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
// about these during lowering.
MaxStoresPerMemcpy  = 0xffffffff;
MaxStoresPerMemmove = 0xffffffff;
MaxStoresPerMemset  = 0xffffffff;

// The expansion for 64-bit division is enormous.
if (AMDGPUBypassSlowDiv)
  addBypassSlowDiv(64, 32);

setTargetDAGCombine({ISD::BITCAST,    ISD::SHL,
                     ISD::SRA,        ISD::SRL,
                     ISD::TRUNCATE,   ISD::MUL,
                     ISD::SMUL_LOHI,  ISD::UMUL_LOHI,
                     ISD::MULHU,      ISD::MULHS,
                     ISD::SELECT,     ISD::SELECT_CC,
                     ISD::STORE,      ISD::FADD,
                     ISD::FSUB,       ISD::FNEG,
                     ISD::FABS,       ISD::AssertZext,
                     ISD::AssertSext, ISD::INTRINSIC_WO_CHAIN});
604}

606bool AMDGPUTargetLowering::mayIgnoreSignedZero(SDValue Op) const {
if (getTargetMachine().Options.NoSignedZerosFPMath)
  return true;

const auto Flags = Op.getNode()->getFlags();
if (Flags.hasNoSignedZeros())
  return true;

return false;
615}

617//===----------------------------------------------------------------------===//
618// Target Information
619//===----------------------------------------------------------------------===//

621LLVM_READNONE__attribute__((__const__))
622static bool fnegFoldsIntoOp(unsigned Opc) {
switch (Opc) {
case ISD::FADD:
case ISD::FSUB:
case ISD::FMUL:
case ISD::FMA:
case ISD::FMAD:
case ISD::FMINNUM:
case ISD::FMAXNUM:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
case ISD::FSIN:
case ISD::FTRUNC:
case ISD::FRINT:
case ISD::FNEARBYINT:
case ISD::FCANONICALIZE:
case AMDGPUISD::RCP:
case AMDGPUISD::RCP_LEGACY:
case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::SIN_HW:
case AMDGPUISD::FMUL_LEGACY:
case AMDGPUISD::FMIN_LEGACY:
case AMDGPUISD::FMAX_LEGACY:
case AMDGPUISD::FMED3:
  // TODO: handle llvm.amdgcn.fma.legacy
  return true;
default:
  return false;
}
651}

653/// \p returns true if the operation will definitely need to use a 64-bit
654/// encoding, and thus will use a VOP3 encoding regardless of the source
655/// modifiers.
656LLVM_READONLY__attribute__((__pure__))
657static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
return N->getNumOperands() > 2 || VT == MVT::f64;
659}

661// Most FP instructions support source modifiers, but this could be refined
662// slightly.
663LLVM_READONLY__attribute__((__pure__))
664static bool hasSourceMods(const SDNode *N) {
if (isa<MemSDNode>(N))
  return false;

switch (N->getOpcode()) {
case ISD::CopyToReg:
case ISD::SELECT:
case ISD::FDIV:
case ISD::FREM:
case ISD::INLINEASM:
case ISD::INLINEASM_BR:
case AMDGPUISD::DIV_SCALE:
case ISD::INTRINSIC_W_CHAIN:

// TODO: Should really be looking at the users of the bitcast. These are
// problematic because bitcasts are used to legalize all stores to integer
// types.
case ISD::BITCAST:
  return false;
case ISD::INTRINSIC_WO_CHAIN: {
  switch (cast<ConstantSDNode>(N->getOperand(0))->getZExtValue()) {
  case Intrinsic::amdgcn_interp_p1:
  case Intrinsic::amdgcn_interp_p2:
  case Intrinsic::amdgcn_interp_mov:
  case Intrinsic::amdgcn_interp_p1_f16:
  case Intrinsic::amdgcn_interp_p2_f16:
    return false;
  default:
    return true;
  }
}
default:
  return true;
}
698}

700bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
                                               unsigned CostThreshold) {
// Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
// it is truly free to use a source modifier in all cases. If there are
// multiple users but for each one will necessitate using VOP3, there will be
// a code size increase. Try to avoid increasing code size unless we know it
// will save on the instruction count.
unsigned NumMayIncreaseSize = 0;
MVT VT = N->getValueType(0).getScalarType().getSimpleVT();

// XXX - Should this limit number of uses to check?
for (const SDNode *U : N->uses()) {
  if (!hasSourceMods(U))
    return false;

  if (!opMustUseVOP3Encoding(U, VT)) {
    if (++NumMayIncreaseSize > CostThreshold)
      return false;
  }
}

return true;
722}

724EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
                                            ISD::NodeType ExtendKind) const {
assert(!VT.isVector() && "only scalar expected")(static_cast <bool> (!VT.isVector() && "only scalar expected"
) ? void (0) : __assert_fail ("!VT.isVector() && \"only scalar expected\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 726, __extension__
 __PRETTY_FUNCTION__));

// Round to the next multiple of 32-bits.
unsigned Size = VT.getSizeInBits();
if (Size <= 32)
  return MVT::i32;
return EVT::getIntegerVT(Context, 32 * ((Size + 31) / 32));
733}

735MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const {
return MVT::i32;
737}

739bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
return true;
741}

743// The backend supports 32 and 64 bit floating point immediates.
744// FIXME: Why are we reporting vectors of FP immediates as legal?
745bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                      bool ForCodeSize) const {
EVT ScalarVT = VT.getScalarType();
return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 ||
       (ScalarVT == MVT::f16 && Subtarget->has16BitInsts()));
750}

752// We don't want to shrink f64 / f32 constants.
753bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
EVT ScalarVT = VT.getScalarType();
return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
756}

758bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N,
                                               ISD::LoadExtType ExtTy,
                                               EVT NewVT) const {
// TODO: This may be worth removing. Check regression tests for diffs.
if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT))
  return false;

unsigned NewSize = NewVT.getStoreSizeInBits();

// If we are reducing to a 32-bit load or a smaller multi-dword load,
// this is always better.
if (NewSize >= 32)
  return true;

EVT OldVT = N->getValueType(0);
unsigned OldSize = OldVT.getStoreSizeInBits();

MemSDNode *MN = cast<MemSDNode>(N);
unsigned AS = MN->getAddressSpace();
// Do not shrink an aligned scalar load to sub-dword.
// Scalar engine cannot do sub-dword loads.
if (OldSize >= 32 && NewSize < 32 && MN->getAlignment() >= 4 &&
    (AS == AMDGPUAS::CONSTANT_ADDRESS ||
     AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
     (isa<LoadSDNode>(N) &&
      AS == AMDGPUAS::GLOBAL_ADDRESS && MN->isInvariant())) &&
    AMDGPUInstrInfo::isUniformMMO(MN->getMemOperand()))
  return false;

// Don't produce extloads from sub 32-bit types. SI doesn't have scalar
// extloads, so doing one requires using a buffer_load. In cases where we
// still couldn't use a scalar load, using the wider load shouldn't really
// hurt anything.

// If the old size already had to be an extload, there's no harm in continuing
// to reduce the width.
return (OldSize < 32);
795}

797bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
                                                 const SelectionDAG &DAG,
                                                 const MachineMemOperand &MMO) const {

assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits())(static_cast <bool> (LoadTy.getSizeInBits() == CastTy.getSizeInBits
()) ? void (0) : __assert_fail ("LoadTy.getSizeInBits() == CastTy.getSizeInBits()"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 801, __extension__
 __PRETTY_FUNCTION__));

if (LoadTy.getScalarType() == MVT::i32)
  return false;

unsigned LScalarSize = LoadTy.getScalarSizeInBits();
unsigned CastScalarSize = CastTy.getScalarSizeInBits();

if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
  return false;

bool Fast = false;
return allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                      CastTy, MMO, &Fast) &&
       Fast;
816}

818// SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
819// profitable with the expansion for 64-bit since it's generally good to
820// speculate things.
821// FIXME: These should really have the size as a parameter.
822bool AMDGPUTargetLowering::isCheapToSpeculateCttz() const {
return true;
824}

826bool AMDGPUTargetLowering::isCheapToSpeculateCtlz() const {
return true;
828}

830bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
switch (N->getOpcode()) {
case ISD::EntryToken:
case ISD::TokenFactor:
  return true;
case ISD::INTRINSIC_WO_CHAIN: {
  unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
  switch (IntrID) {
  case Intrinsic::amdgcn_readfirstlane:
  case Intrinsic::amdgcn_readlane:
    return true;
  }
  return false;
}
case ISD::LOAD:
  if (cast<LoadSDNode>(N)->getMemOperand()->getAddrSpace() ==
      AMDGPUAS::CONSTANT_ADDRESS_32BIT)
    return true;
  return false;
case AMDGPUISD::SETCC: // ballot-style instruction
  return true;
}
return false;
853}

855SDValue AMDGPUTargetLowering::getNegatedExpression(
  SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize,
  NegatibleCost &Cost, unsigned Depth) const {

switch (Op.getOpcode()) {
case ISD::FMA:
case ISD::FMAD: {
  // Negating a fma is not free if it has users without source mods.
  if (!allUsesHaveSourceMods(Op.getNode()))
    return SDValue();
  break;
}
default:
  break;
}

return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
                                            ForCodeSize, Cost, Depth);
873}

875//===---------------------------------------------------------------------===//
876// Target Properties
877//===---------------------------------------------------------------------===//

879bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
assert(VT.isFloatingPoint())(static_cast <bool> (VT.isFloatingPoint()) ? void (0) :
 __assert_fail ("VT.isFloatingPoint()", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 880, __extension__ __PRETTY_FUNCTION__));

// Packed operations do not have a fabs modifier.
return VT == MVT::f32 || VT == MVT::f64 ||
       (Subtarget->has16BitInsts() && VT == MVT::f16);
885}

887bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
assert(VT.isFloatingPoint())(static_cast <bool> (VT.isFloatingPoint()) ? void (0) :
 __assert_fail ("VT.isFloatingPoint()", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 888, __extension__ __PRETTY_FUNCTION__));
// Report this based on the end legalized type.
VT = VT.getScalarType();
return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16;
892}

894bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(EVT MemVT,
                                                       unsigned NumElem,
                                                       unsigned AS) const {
return true;
898}

900bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
// There are few operations which truly have vector input operands. Any vector
// operation is going to involve operations on each component, and a
// build_vector will be a copy per element, so it always makes sense to use a
// build_vector input in place of the extracted element to avoid a copy into a
// super register.
//
// We should probably only do this if all users are extracts only, but this
// should be the common case.
return true;
910}

912bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
// Truncate is just accessing a subregister.

unsigned SrcSize = Source.getSizeInBits();
unsigned DestSize = Dest.getSizeInBits();

return DestSize < SrcSize && DestSize % 32 == 0 ;
919}

921bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
// Truncate is just accessing a subregister.

unsigned SrcSize = Source->getScalarSizeInBits();
unsigned DestSize = Dest->getScalarSizeInBits();

if (DestSize== 16 && Subtarget->has16BitInsts())
  return SrcSize >= 32;

return DestSize < SrcSize && DestSize % 32 == 0;
931}

933bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
unsigned SrcSize = Src->getScalarSizeInBits();
unsigned DestSize = Dest->getScalarSizeInBits();

if (SrcSize == 16 && Subtarget->has16BitInsts())
  return DestSize >= 32;

return SrcSize == 32 && DestSize == 64;
941}

943bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
// Any register load of a 64-bit value really requires 2 32-bit moves. For all
// practical purposes, the extra mov 0 to load a 64-bit is free.  As used,
// this will enable reducing 64-bit operations the 32-bit, which is always
// good.

if (Src == MVT::i16)
  return Dest == MVT::i32 ||Dest == MVT::i64 ;

return Src == MVT::i32 && Dest == MVT::i64;
953}

955bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
return isZExtFree(Val.getValueType(), VT2);
957}

959bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
// There aren't really 64-bit registers, but pairs of 32-bit ones and only a
// limited number of native 64-bit operations. Shrinking an operation to fit
// in a single 32-bit register should always be helpful. As currently used,
// this is much less general than the name suggests, and is only used in
// places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
// not profitable, and may actually be harmful.
return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
967}

969//===---------------------------------------------------------------------===//
970// TargetLowering Callbacks
971//===---------------------------------------------------------------------===//

973CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
                                                bool IsVarArg) {
switch (CC) {
case CallingConv::AMDGPU_VS:
case CallingConv::AMDGPU_GS:
case CallingConv::AMDGPU_PS:
case CallingConv::AMDGPU_CS:
case CallingConv::AMDGPU_HS:
case CallingConv::AMDGPU_ES:
case CallingConv::AMDGPU_LS:
  return CC_AMDGPU;
case CallingConv::C:
case CallingConv::Fast:
case CallingConv::Cold:
  return CC_AMDGPU_Func;
case CallingConv::AMDGPU_Gfx:
  return CC_SI_Gfx;
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
default:
  report_fatal_error("Unsupported calling convention for call");
}
995}

997CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
                                                  bool IsVarArg) {
switch (CC) {
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
  llvm_unreachable("kernels should not be handled here")::llvm::llvm_unreachable_internal("kernels should not be handled here"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1002);
case CallingConv::AMDGPU_VS:
case CallingConv::AMDGPU_GS:
case CallingConv::AMDGPU_PS:
case CallingConv::AMDGPU_CS:
case CallingConv::AMDGPU_HS:
case CallingConv::AMDGPU_ES:
case CallingConv::AMDGPU_LS:
  return RetCC_SI_Shader;
case CallingConv::AMDGPU_Gfx:
  return RetCC_SI_Gfx;
case CallingConv::C:
case CallingConv::Fast:
case CallingConv::Cold:
  return RetCC_AMDGPU_Func;
default:
  report_fatal_error("Unsupported calling convention.");
}
1020}

1022/// The SelectionDAGBuilder will automatically promote function arguments
1023/// with illegal types.  However, this does not work for the AMDGPU targets
1024/// since the function arguments are stored in memory as these illegal types.
1025/// In order to handle this properly we need to get the original types sizes
1026/// from the LLVM IR Function and fixup the ISD:InputArg values before
1027/// passing them to AnalyzeFormalArguments()

1029/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
1030/// input values across multiple registers.  Each item in the Ins array
1031/// represents a single value that will be stored in registers.  Ins[x].VT is
1032/// the value type of the value that will be stored in the register, so
1033/// whatever SDNode we lower the argument to needs to be this type.
1034///
1035/// In order to correctly lower the arguments we need to know the size of each
1036/// argument.  Since Ins[x].VT gives us the size of the register that will
1037/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
1038/// for the original function argument so that we can deduce the correct memory
1039/// type to use for Ins[x].  In most cases the correct memory type will be
1040/// Ins[x].ArgVT.  However, this will not always be the case.  If, for example,
1041/// we have a kernel argument of type v8i8, this argument will be split into
1042/// 8 parts and each part will be represented by its own item in the Ins array.
1043/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
1044/// the argument before it was split.  From this, we deduce that the memory type
1045/// for each individual part is i8.  We pass the memory type as LocVT to the
1046/// calling convention analysis function and the register type (Ins[x].VT) as
1047/// the ValVT.
1048void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
CCState &State,
const SmallVectorImpl<ISD::InputArg> &Ins) const {
const MachineFunction &MF = State.getMachineFunction();
const Function &Fn = MF.getFunction();
LLVMContext &Ctx = Fn.getParent()->getContext();
const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(MF);
const unsigned ExplicitOffset = ST.getExplicitKernelArgOffset(Fn);
CallingConv::ID CC = Fn.getCallingConv();

Align MaxAlign = Align(1);
uint64_t ExplicitArgOffset = 0;
const DataLayout &DL = Fn.getParent()->getDataLayout();

unsigned InIndex = 0;

for (const Argument &Arg : Fn.args()) {
  const bool IsByRef = Arg.hasByRefAttr();
  Type *BaseArgTy = Arg.getType();
  Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
  MaybeAlign Alignment = IsByRef ? Arg.getParamAlign() : None;
  if (!Alignment)
    Alignment = DL.getABITypeAlign(MemArgTy);
  MaxAlign = max(Alignment, MaxAlign);
  uint64_t AllocSize = DL.getTypeAllocSize(MemArgTy);

  uint64_t ArgOffset = alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset;
  ExplicitArgOffset = alignTo(ExplicitArgOffset, Alignment) + AllocSize;

  // We're basically throwing away everything passed into us and starting over
  // to get accurate in-memory offsets. The "PartOffset" is completely useless
  // to us as computed in Ins.
  //
  // We also need to figure out what type legalization is trying to do to get
  // the correct memory offsets.

  SmallVector<EVT, 16> ValueVTs;
  SmallVector<uint64_t, 16> Offsets;
  ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, &Offsets, ArgOffset);

  for (unsigned Value = 0, NumValues = ValueVTs.size();
       Value != NumValues; ++Value) {
    uint64_t BasePartOffset = Offsets[Value];

    EVT ArgVT = ValueVTs[Value];
    EVT MemVT = ArgVT;
    MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT);
    unsigned NumRegs = getNumRegistersForCallingConv(Ctx, CC, ArgVT);

    if (NumRegs == 1) {
      // This argument is not split, so the IR type is the memory type.
      if (ArgVT.isExtended()) {
        // We have an extended type, like i24, so we should just use the
        // register type.
        MemVT = RegisterVT;
      } else {
        MemVT = ArgVT;
      }
    } else if (ArgVT.isVector() && RegisterVT.isVector() &&
               ArgVT.getScalarType() == RegisterVT.getScalarType()) {
      assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements())(static_cast <bool> (ArgVT.getVectorNumElements() > RegisterVT
.getVectorNumElements()) ? void (0) : __assert_fail ("ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements()"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1108, __extension__
 __PRETTY_FUNCTION__));
      // We have a vector value which has been split into a vector with
      // the same scalar type, but fewer elements.  This should handle
      // all the floating-point vector types.
      MemVT = RegisterVT;
    } else if (ArgVT.isVector() &&
               ArgVT.getVectorNumElements() == NumRegs) {
      // This arg has been split so that each element is stored in a separate
      // register.
      MemVT = ArgVT.getScalarType();
    } else if (ArgVT.isExtended()) {
      // We have an extended type, like i65.
      MemVT = RegisterVT;
    } else {
      unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
      assert(ArgVT.getStoreSizeInBits() % NumRegs == 0)(static_cast <bool> (ArgVT.getStoreSizeInBits() % NumRegs
 == 0) ? void (0) : __assert_fail ("ArgVT.getStoreSizeInBits() % NumRegs == 0"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1123, __extension__
 __PRETTY_FUNCTION__));
      if (RegisterVT.isInteger()) {
        MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits);
      } else if (RegisterVT.isVector()) {
        assert(!RegisterVT.getScalarType().isFloatingPoint())(static_cast <bool> (!RegisterVT.getScalarType().isFloatingPoint
()) ? void (0) : __assert_fail ("!RegisterVT.getScalarType().isFloatingPoint()"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1127, __extension__
 __PRETTY_FUNCTION__));
        unsigned NumElements = RegisterVT.getVectorNumElements();
        assert(MemoryBits % NumElements == 0)(static_cast <bool> (MemoryBits % NumElements == 0) ? void
 (0) : __assert_fail ("MemoryBits % NumElements == 0", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 1129, __extension__ __PRETTY_FUNCTION__));
        // This vector type has been split into another vector type with
        // a different elements size.
        EVT ScalarVT = EVT::getIntegerVT(State.getContext(),
                                         MemoryBits / NumElements);
        MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements);
      } else {
        llvm_unreachable("cannot deduce memory type.")::llvm::llvm_unreachable_internal("cannot deduce memory type."
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1136);
      }
    }

    // Convert one element vectors to scalar.
    if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
      MemVT = MemVT.getScalarType();

    // Round up vec3/vec5 argument.
    if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
      assert(MemVT.getVectorNumElements() == 3 ||(static_cast <bool> (MemVT.getVectorNumElements() == 3 ||
 MemVT.getVectorNumElements() == 5) ? void (0) : __assert_fail
 ("MemVT.getVectorNumElements() == 3 || MemVT.getVectorNumElements() == 5"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1147, __extension__
 __PRETTY_FUNCTION__))
             MemVT.getVectorNumElements() == 5)(static_cast <bool> (MemVT.getVectorNumElements() == 3 ||
 MemVT.getVectorNumElements() == 5) ? void (0) : __assert_fail
 ("MemVT.getVectorNumElements() == 3 || MemVT.getVectorNumElements() == 5"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1147, __extension__
 __PRETTY_FUNCTION__));
      MemVT = MemVT.getPow2VectorType(State.getContext());
    } else if (!MemVT.isSimple() && !MemVT.isVector()) {
      MemVT = MemVT.getRoundIntegerType(State.getContext());
    }

    unsigned PartOffset = 0;
    for (unsigned i = 0; i != NumRegs; ++i) {
      State.addLoc(CCValAssign::getCustomMem(InIndex++, RegisterVT,
                                             BasePartOffset + PartOffset,
                                             MemVT.getSimpleVT(),
                                             CCValAssign::Full));
      PartOffset += MemVT.getStoreSize();
    }
  }
}
1163}

1165SDValue AMDGPUTargetLowering::LowerReturn(
SDValue Chain, CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
const SDLoc &DL, SelectionDAG &DAG) const {
// FIXME: Fails for r600 tests
//assert(!isVarArg && Outs.empty() && OutVals.empty() &&
// "wave terminate should not have return values");
return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
1175}

1177//===---------------------------------------------------------------------===//
1178// Target specific lowering
1179//===---------------------------------------------------------------------===//

1181/// Selects the correct CCAssignFn for a given CallingConvention value.
1182CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
                                                  bool IsVarArg) {
return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1185}

1187CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
                                                    bool IsVarArg) {
return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1190}

1192SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
                                                SelectionDAG &DAG,
                                                MachineFrameInfo &MFI,
                                                int ClobberedFI) const {
SmallVector<SDValue, 8> ArgChains;
int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;

// Include the original chain at the beginning of the list. When this is
// used by target LowerCall hooks, this helps legalize find the
// CALLSEQ_BEGIN node.
ArgChains.push_back(Chain);

// Add a chain value for each stack argument corresponding
for (SDNode *U : DAG.getEntryNode().getNode()->uses()) {
  if (LoadSDNode *L = dyn_cast<LoadSDNode>(U)) {
    if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) {
      if (FI->getIndex() < 0) {
        int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
        int64_t InLastByte = InFirstByte;
        InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;

        if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
            (FirstByte <= InFirstByte && InFirstByte <= LastByte))
          ArgChains.push_back(SDValue(L, 1));
      }
    }
  }
}

// Build a tokenfactor for all the chains.
return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
1224}

1226SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
                                               SmallVectorImpl<SDValue> &InVals,
                                               StringRef Reason) const {
SDValue Callee = CLI.Callee;
SelectionDAG &DAG = CLI.DAG;

const Function &Fn = DAG.getMachineFunction().getFunction();

StringRef FuncName("<unknown>");

if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee))
  FuncName = G->getSymbol();
else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
  FuncName = G->getGlobal()->getName();

DiagnosticInfoUnsupported NoCalls(
  Fn, Reason + FuncName, CLI.DL.getDebugLoc());
DAG.getContext()->diagnose(NoCalls);

if (!CLI.IsTailCall) {
  for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
    InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
}

return DAG.getEntryNode();
1251}

1253SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
                                      SmallVectorImpl<SDValue> &InVals) const {
return lowerUnhandledCall(CLI, InVals, "unsupported call to function ");
1256}

1258SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                    SelectionDAG &DAG) const {
const Function &Fn = DAG.getMachineFunction().getFunction();

DiagnosticInfoUnsupported NoDynamicAlloca(Fn, "unsupported dynamic alloca",
                                          SDLoc(Op).getDebugLoc());
DAG.getContext()->diagnose(NoDynamicAlloca);
auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
return DAG.getMergeValues(Ops, SDLoc());
1267}

1269SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
                                           SelectionDAG &DAG) const {
switch (Op.getOpcode()) {
default:
  Op->print(errs(), &DAG);
  llvm_unreachable("Custom lowering code for this "::llvm::llvm_unreachable_internal("Custom lowering code for this "
 "instruction is not implemented yet!", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 1275)
                   "instruction is not implemented yet!")::llvm::llvm_unreachable_internal("Custom lowering code for this "
 "instruction is not implemented yet!", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 1275);
  break;
case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
case ISD::FREM: return LowerFREM(Op, DAG);
case ISD::FCEIL: return LowerFCEIL(Op, DAG);
case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
case ISD::FRINT: return LowerFRINT(Op, DAG);
case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
case ISD::FROUND: return LowerFROUND(Op, DAG);
case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
case ISD::FLOG:
  return LowerFLOG(Op, DAG, numbers::ln2f);
case ISD::FLOG10:
  return LowerFLOG(Op, DAG, numbers::ln2f / numbers::ln10f);
case ISD::FEXP:
  return lowerFEXP(Op, DAG);
case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:
  return LowerFP_TO_INT(Op, DAG);
case ISD::CTTZ:
case ISD::CTTZ_ZERO_UNDEF:
case ISD::CTLZ:
case ISD::CTLZ_ZERO_UNDEF:
  return LowerCTLZ_CTTZ(Op, DAG);
case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
}
return Op;
1309}

1311void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
                                            SmallVectorImpl<SDValue> &Results,
                                            SelectionDAG &DAG) const {
switch (N->getOpcode()) {
case ISD::SIGN_EXTEND_INREG:
  // Different parts of legalization seem to interpret which type of
  // sign_extend_inreg is the one to check for custom lowering. The extended
  // from type is what really matters, but some places check for custom
  // lowering of the result type. This results in trying to use
  // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
  // nothing here and let the illegal result integer be handled normally.
  return;
default:
  return;
}
1326}

1328SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
                                               SDValue Op,
                                               SelectionDAG &DAG) const {

const DataLayout &DL = DAG.getDataLayout();
GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
const GlobalValue *GV = G->getGlobal();

if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
    G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
  if (!MFI->isModuleEntryFunction() &&
      !GV->getName().equals("llvm.amdgcn.module.lds")) {
    SDLoc DL(Op);
    const Function &Fn = DAG.getMachineFunction().getFunction();
    DiagnosticInfoUnsupported BadLDSDecl(
      Fn, "local memory global used by non-kernel function",
      DL.getDebugLoc(), DS_Warning);
    DAG.getContext()->diagnose(BadLDSDecl);

    // We currently don't have a way to correctly allocate LDS objects that
    // aren't directly associated with a kernel. We do force inlining of
    // functions that use local objects. However, if these dead functions are
    // not eliminated, we don't want a compile time error. Just emit a warning
    // and a trap, since there should be no callable path here.
    SDValue Trap = DAG.getNode(ISD::TRAP, DL, MVT::Other, DAG.getEntryNode());
    SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
                                      Trap, DAG.getRoot());
    DAG.setRoot(OutputChain);
    return DAG.getUNDEF(Op.getValueType());
  }

  // XXX: What does the value of G->getOffset() mean?
  assert(G->getOffset() == 0 &&(static_cast <bool> (G->getOffset() == 0 && "Do not know what to do with an non-zero offset"
) ? void (0) : __assert_fail ("G->getOffset() == 0 && \"Do not know what to do with an non-zero offset\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1361, __extension__
 __PRETTY_FUNCTION__))
       "Do not know what to do with an non-zero offset")(static_cast <bool> (G->getOffset() == 0 && "Do not know what to do with an non-zero offset"
) ? void (0) : __assert_fail ("G->getOffset() == 0 && \"Do not know what to do with an non-zero offset\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1361, __extension__
 __PRETTY_FUNCTION__));

  // TODO: We could emit code to handle the initialization somewhere.
  // We ignore the initializer for now and legalize it to allow selection.
  // The initializer will anyway get errored out during assembly emission.
  unsigned Offset = MFI->allocateLDSGlobal(DL, *cast<GlobalVariable>(GV));
  return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
}
return SDValue();
1370}

1372SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
                                                SelectionDAG &DAG) const {
SmallVector<SDValue, 8> Args;

EVT VT = Op.getValueType();
if (VT == MVT::v4i16 || VT == MVT::v4f16) {
  SDLoc SL(Op);
  SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0));
  SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1));

  SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi });
  return DAG.getNode(ISD::BITCAST, SL, VT, BV);
}

for (const SDUse &U : Op->ops())
  DAG.ExtractVectorElements(U.get(), Args);

return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
1390}

1392SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
                                                   SelectionDAG &DAG) const {

SmallVector<SDValue, 8> Args;
unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
EVT VT = Op.getValueType();
EVT SrcVT = Op.getOperand(0).getValueType();

// For these types, we have some TableGen patterns except if the index is 1
if (((SrcVT == MVT::v4f16 && VT == MVT::v2f16) ||
     (SrcVT == MVT::v4i16 && VT == MVT::v2i16)) &&
    Start != 1)
  return Op;

if (((SrcVT == MVT::v8f16 && VT == MVT::v4f16) ||
     (SrcVT == MVT::v8i16 && VT == MVT::v4i16)) &&
    (Start == 0 || Start == 4))
  return Op;

DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
                          VT.getVectorNumElements());

return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
1415}

1417/// Generate Min/Max node
1418SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
                                                 SDValue LHS, SDValue RHS,
                                                 SDValue True, SDValue False,
                                                 SDValue CC,
                                                 DAGCombinerInfo &DCI) const {
if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
switch (CCOpcode) {
case ISD::SETOEQ:
case ISD::SETONE:
case ISD::SETUNE:
case ISD::SETNE:
case ISD::SETUEQ:
case ISD::SETEQ:
case ISD::SETFALSE:
case ISD::SETFALSE2:
case ISD::SETTRUE:
case ISD::SETTRUE2:
case ISD::SETUO:
case ISD::SETO:
  break;
case ISD::SETULE:
case ISD::SETULT: {
  if (LHS == True)
    return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
  return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
}
case ISD::SETOLE:
case ISD::SETOLT:
case ISD::SETLE:
case ISD::SETLT: {
  // Ordered. Assume ordered for undefined.

  // Only do this after legalization to avoid interfering with other combines
  // which might occur.
  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
      !DCI.isCalledByLegalizer())
    return SDValue();

  // We need to permute the operands to get the correct NaN behavior. The
  // selected operand is the second one based on the failing compare with NaN,
  // so permute it based on the compare type the hardware uses.
  if (LHS == True)
    return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
  return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
}
case ISD::SETUGE:
case ISD::SETUGT: {
  if (LHS == True)
    return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
  return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
}
case ISD::SETGT:
case ISD::SETGE:
case ISD::SETOGE:
case ISD::SETOGT: {
  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
      !DCI.isCalledByLegalizer())
    return SDValue();

  if (LHS == True)
    return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
  return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
}
case ISD::SETCC_INVALID:
  llvm_unreachable("Invalid setcc condcode!")::llvm::llvm_unreachable_internal("Invalid setcc condcode!", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 1486);
}
return SDValue();
1489}

1491std::pair<SDValue, SDValue>
1492AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);

SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);

const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
const SDValue One = DAG.getConstant(1, SL, MVT::i32);

SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);

return std::make_pair(Lo, Hi);
1504}

1506SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);

SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1512}

1514SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);

SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
const SDValue One = DAG.getConstant(1, SL, MVT::i32);
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1520}

1522// Split a vector type into two parts. The first part is a power of two vector.
1523// The second part is whatever is left over, and is a scalar if it would
1524// otherwise be a 1-vector.
1525std::pair<EVT, EVT>
1526AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
EVT LoVT, HiVT;
EVT EltVT = VT.getVectorElementType();
unsigned NumElts = VT.getVectorNumElements();
unsigned LoNumElts = PowerOf2Ceil((NumElts + 1) / 2);
LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts);
HiVT = NumElts - LoNumElts == 1
           ? EltVT
           : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts);
return std::make_pair(LoVT, HiVT);
1536}

1538// Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1539// scalar.
1540std::pair<SDValue, SDValue>
1541AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
                                const EVT &LoVT, const EVT &HiVT,
                                SelectionDAG &DAG) const {
assert(LoVT.getVectorNumElements() +(static_cast <bool> (LoVT.getVectorNumElements() + (HiVT
.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType
().getVectorNumElements() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorNumElements() + (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType().getVectorNumElements() && \"More vector elements requested than available!\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1547, __extension__
 __PRETTY_FUNCTION__))
               (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=(static_cast <bool> (LoVT.getVectorNumElements() + (HiVT
.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType
().getVectorNumElements() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorNumElements() + (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType().getVectorNumElements() && \"More vector elements requested than available!\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1547, __extension__
 __PRETTY_FUNCTION__))
           N.getValueType().getVectorNumElements() &&(static_cast <bool> (LoVT.getVectorNumElements() + (HiVT
.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType
().getVectorNumElements() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorNumElements() + (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType().getVectorNumElements() && \"More vector elements requested than available!\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1547, __extension__
 __PRETTY_FUNCTION__))
       "More vector elements requested than available!")(static_cast <bool> (LoVT.getVectorNumElements() + (HiVT
.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType
().getVectorNumElements() && "More vector elements requested than available!"
) ? void (0) : __assert_fail ("LoVT.getVectorNumElements() + (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <= N.getValueType().getVectorNumElements() && \"More vector elements requested than available!\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1547, __extension__
 __PRETTY_FUNCTION__));
SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
                         DAG.getVectorIdxConstant(0, DL));
SDValue Hi = DAG.getNode(
    HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL,
    HiVT, N, DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), DL));
return std::make_pair(Lo, Hi);
1554}

1556SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
                                            SelectionDAG &DAG) const {
LoadSDNode *Load = cast<LoadSDNode>(Op);
EVT VT = Op.getValueType();
SDLoc SL(Op);


// If this is a 2 element vector, we really want to scalarize and not create
// weird 1 element vectors.
if (VT.getVectorNumElements() == 2) {
  SDValue Ops[2];
  std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG);
  return DAG.getMergeValues(Ops, SL);
}

SDValue BasePtr = Load->getBasePtr();
EVT MemVT = Load->getMemoryVT();

const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();

EVT LoVT, HiVT;
EVT LoMemVT, HiMemVT;
SDValue Lo, Hi;

std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG);

unsigned Size = LoMemVT.getStoreSize();
unsigned BaseAlign = Load->getAlignment();
unsigned HiAlign = MinAlign(BaseAlign, Size);

SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
                                Load->getChain(), BasePtr, SrcValue, LoMemVT,
                                BaseAlign, Load->getMemOperand()->getFlags());
SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Size));
SDValue HiLoad =
    DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
                   HiPtr, SrcValue.getWithOffset(LoMemVT.getStoreSize()),
                   HiMemVT, HiAlign, Load->getMemOperand()->getFlags());

SDValue Join;
if (LoVT == HiVT) {
  // This is the case that the vector is power of two so was evenly split.
  Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
} else {
  Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad,
                     DAG.getVectorIdxConstant(0, SL));
  Join = DAG.getNode(
      HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL,
      VT, Join, HiLoad,
      DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), SL));
}

SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
                                   LoLoad.getValue(1), HiLoad.getValue(1))};

return DAG.getMergeValues(Ops, SL);
1614}

1616SDValue AMDGPUTargetLowering::WidenOrSplitVectorLoad(SDValue Op,
                                                   SelectionDAG &DAG) const {
LoadSDNode *Load = cast<LoadSDNode>(Op);
EVT VT = Op.getValueType();
SDValue BasePtr = Load->getBasePtr();
EVT MemVT = Load->getMemoryVT();
SDLoc SL(Op);
const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
unsigned BaseAlign = Load->getAlignment();
unsigned NumElements = MemVT.getVectorNumElements();

// Widen from vec3 to vec4 when the load is at least 8-byte aligned
// or 16-byte fully dereferenceable. Otherwise, split the vector load.
if (NumElements != 3 ||
    (BaseAlign < 8 &&
     !SrcValue.isDereferenceable(16, *DAG.getContext(), DAG.getDataLayout())))
  return SplitVectorLoad(Op, DAG);

assert(NumElements == 3)(static_cast <bool> (NumElements == 3) ? void (0) : __assert_fail
 ("NumElements == 3", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 1634, __extension__ __PRETTY_FUNCTION__));

EVT WideVT =
    EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
EVT WideMemVT =
    EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 4);
SDValue WideLoad = DAG.getExtLoad(
    Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue,
    WideMemVT, BaseAlign, Load->getMemOperand()->getFlags());
return DAG.getMergeValues(
    {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad,
                 DAG.getVectorIdxConstant(0, SL)),
     WideLoad.getValue(1)},
    SL);
1648}

1650SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
                                             SelectionDAG &DAG) const {
StoreSDNode *Store = cast<StoreSDNode>(Op);
SDValue Val = Store->getValue();
EVT VT = Val.getValueType();

// If this is a 2 element vector, we really want to scalarize and not create
// weird 1 element vectors.
if (VT.getVectorNumElements() == 2)
  return scalarizeVectorStore(Store, DAG);

EVT MemVT = Store->getMemoryVT();
SDValue Chain = Store->getChain();
SDValue BasePtr = Store->getBasePtr();
SDLoc SL(Op);

EVT LoVT, HiVT;
EVT LoMemVT, HiMemVT;
SDValue Lo, Hi;

std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG);

SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());

const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
unsigned BaseAlign = Store->getAlignment();
unsigned Size = LoMemVT.getStoreSize();
unsigned HiAlign = MinAlign(BaseAlign, Size);

SDValue LoStore =
    DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
                      Store->getMemOperand()->getFlags());
SDValue HiStore =
    DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size),
                      HiMemVT, HiAlign, Store->getMemOperand()->getFlags());

return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1689}

1691// This is a shortcut for integer division because we have fast i32<->f32
1692// conversions, and fast f32 reciprocal instructions. The fractional part of a
1693// float is enough to accurately represent up to a 24-bit signed integer.
1694SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
                                          bool Sign) const {
SDLoc DL(Op);
EVT VT = Op.getValueType();
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
MVT IntVT = MVT::i32;
MVT FltVT = MVT::f32;

unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS);
if (LHSSignBits < 9)
  return SDValue();

unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS);
if (RHSSignBits < 9)
  return SDValue();

unsigned BitSize = VT.getSizeInBits();
unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
unsigned DivBits = BitSize - SignBits;
if (Sign)
  ++DivBits;

ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;

SDValue jq = DAG.getConstant(1, DL, IntVT);

if (Sign) {
  // char|short jq = ia ^ ib;
  jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);

  // jq = jq >> (bitsize - 2)
  jq = DAG.getNode(ISD::SRA, DL, VT, jq,
                   DAG.getConstant(BitSize - 2, DL, VT));

  // jq = jq | 0x1
  jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
}

// int ia = (int)LHS;
SDValue ia = LHS;

// int ib, (int)RHS;
SDValue ib = RHS;

// float fa = (float)ia;
SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);

// float fb = (float)ib;
SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);

SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
                         fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));

// fq = trunc(fq);
fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);

// float fqneg = -fq;
SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);

MachineFunction &MF = DAG.getMachineFunction();
const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();

// float fr = mad(fqneg, fb, fa);
unsigned OpCode = !Subtarget->hasMadMacF32Insts() ?
                  (unsigned)ISD::FMA :
                  !MFI->getMode().allFP32Denormals() ?
                  (unsigned)ISD::FMAD :
                  (unsigned)AMDGPUISD::FMAD_FTZ;
SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);

// int iq = (int)fq;
SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);

// fr = fabs(fr);
fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);

// fb = fabs(fb);
fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

// int cv = fr >= fb;
SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);

// jq = (cv ? jq : 0);
jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));

// dst = iq + jq;
SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);

// Rem needs compensation, it's easier to recompute it
SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);

// Truncate to number of bits this divide really is.
if (Sign) {
  SDValue InRegSize
    = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits));
  Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
  Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
} else {
  SDValue TruncMask = DAG.getConstant((UINT64_C(1)1UL << DivBits) - 1, DL, VT);
  Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
  Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
}

return DAG.getMergeValues({ Div, Rem }, DL);
1803}

1805void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
                                    SelectionDAG &DAG,
                                    SmallVectorImpl<SDValue> &Results) const {
SDLoc DL(Op);
EVT VT = Op.getValueType();

assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64")(static_cast <bool> (VT == MVT::i64 && "LowerUDIVREM64 expects an i64"
) ? void (0) : __assert_fail ("VT == MVT::i64 && \"LowerUDIVREM64 expects an i64\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 1811, __extension__
 __PRETTY_FUNCTION__));

EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());

SDValue One = DAG.getConstant(1, DL, HalfVT);
SDValue Zero = DAG.getConstant(0, DL, HalfVT);

//HiLo split
SDValue LHS = Op.getOperand(0);
SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One);

SDValue RHS = Op.getOperand(1);
SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One);

if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
    DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {

  SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
                            LHS_Lo, RHS_Lo);

  SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
  SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});

  Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
  Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
  return;
}

if (isTypeLegal(MVT::i64)) {
  // The algorithm here is based on ideas from "Software Integer Division",
  // Tom Rodeheffer, August 2008.

  MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

  // Compute denominator reciprocal.
  unsigned FMAD = !Subtarget->hasMadMacF32Insts() ?
                  (unsigned)ISD::FMA :
                  !MFI->getMode().allFP32Denormals() ?
                  (unsigned)ISD::FMAD :
                  (unsigned)AMDGPUISD::FMAD_FTZ;

  SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
  SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
  SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
    DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
    Cvt_Lo);
  SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
  SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
    DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
  SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
    DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
  SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
  SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
    DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
    Mul1);
  SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
  SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
  SDValue Rcp64 = DAG.getBitcast(VT,
                      DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));

  SDValue Zero64 = DAG.getConstant(0, DL, VT);
  SDValue One64  = DAG.getConstant(1, DL, VT);
  SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
  SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);

  // First round of UNR (Unsigned integer Newton-Raphson).
  SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
  SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
  SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
  SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
                                  Zero);
  SDValue Mulhi1_Hi =
      DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1, One);
  SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
                                Mulhi1_Lo, Zero1);
  SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
                                Mulhi1_Hi, Add1_Lo.getValue(1));
  SDValue Add1 = DAG.getBitcast(VT,
                      DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));

  // Second round of UNR.
  SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
  SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
  SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
                                  Zero);
  SDValue Mulhi2_Hi =
      DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2, One);
  SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
                                Mulhi2_Lo, Zero1);
  SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Hi,
                                Mulhi2_Hi, Add2_Lo.getValue(1));
  SDValue Add2 = DAG.getBitcast(VT,
                      DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));

  SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);

  SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);

  SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero);
  SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One);
  SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
                                Mul3_Lo, Zero1);
  SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
                                Mul3_Hi, Sub1_Lo.getValue(1));
  SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
  SDValue Sub1 = DAG.getBitcast(VT,
                      DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));

  SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
  SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
                               ISD::SETUGE);
  SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
                               ISD::SETUGE);
  SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);

  // TODO: Here and below portions of the code can be enclosed into if/endif.
  // Currently control flow is unconditional and we have 4 selects after
  // potential endif to substitute PHIs.

  // if C3 != 0 ...
  SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
                                RHS_Lo, Zero1);
  SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
                                RHS_Hi, Sub1_Lo.getValue(1));
  SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
                                Zero, Sub2_Lo.getValue(1));
  SDValue Sub2 = DAG.getBitcast(VT,
                      DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));

  SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);

  SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
                               ISD::SETUGE);
  SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
                               ISD::SETUGE);
  SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);

  // if (C6 != 0)
  SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);

  SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
                                RHS_Lo, Zero1);
  SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
                                RHS_Hi, Sub2_Lo.getValue(1));
  SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
                                Zero, Sub3_Lo.getValue(1));
  SDValue Sub3 = DAG.getBitcast(VT,
                      DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));

  // endif C6
  // endif C3

  SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
  SDValue Div  = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);

  SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
  SDValue Rem  = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);

  Results.push_back(Div);
  Results.push_back(Rem);

  return;
}

// r600 expandion.
// Get Speculative values
SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);

SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);

SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
SDValue DIV_Lo = Zero;

const unsigned halfBitWidth = HalfVT.getSizeInBits();

for (unsigned i = 0; i < halfBitWidth; ++i) {
  const unsigned bitPos = halfBitWidth - i - 1;
  SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
  // Get value of high bit
  SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
  HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One);
  HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);

  // Shift
  REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
  // Add LHS high bit
  REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);

  SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
  SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);

  DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);

  // Update REM
  SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
  REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
}

SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
Results.push_back(DIV);
Results.push_back(REM);
2019}

2021SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
                                         SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT VT = Op.getValueType();

if (VT == MVT::i64) {
  SmallVector<SDValue, 2> Results;
  LowerUDIVREM64(Op, DAG, Results);
  return DAG.getMergeValues(Results, DL);
}

if (VT == MVT::i32) {
  if (SDValue Res = LowerDIVREM24(Op, DAG, false))
    return Res;
}

SDValue X = Op.getOperand(0);
SDValue Y = Op.getOperand(1);

// See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
// algorithm used here.

// Initial estimate of inv(y).
SDValue Z = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Y);

// One round of UNR.
SDValue NegY = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Y);
SDValue NegYZ = DAG.getNode(ISD::MUL, DL, VT, NegY, Z);
Z = DAG.getNode(ISD::ADD, DL, VT, Z,
                DAG.getNode(ISD::MULHU, DL, VT, Z, NegYZ));

// Quotient/remainder estimate.
SDValue Q = DAG.getNode(ISD::MULHU, DL, VT, X, Z);
SDValue R =
    DAG.getNode(ISD::SUB, DL, VT, X, DAG.getNode(ISD::MUL, DL, VT, Q, Y));

// First quotient/remainder refinement.
EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
SDValue One = DAG.getConstant(1, DL, VT);
SDValue Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
                DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
                DAG.getNode(ISD::SUB, DL, VT, R, Y), R);

// Second quotient/remainder refinement.
Cond = DAG.getSetCC(DL, CCVT, R, Y, ISD::SETUGE);
Q = DAG.getNode(ISD::SELECT, DL, VT, Cond,
                DAG.getNode(ISD::ADD, DL, VT, Q, One), Q);
R = DAG.getNode(ISD::SELECT, DL, VT, Cond,
                DAG.getNode(ISD::SUB, DL, VT, R, Y), R);

return DAG.getMergeValues({Q, R}, DL);
2074}

2076SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
                                         SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT VT = Op.getValueType();

SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);

SDValue Zero = DAG.getConstant(0, DL, VT);
SDValue NegOne = DAG.getConstant(-1, DL, VT);

if (VT == MVT::i32) {
  if (SDValue Res = LowerDIVREM24(Op, DAG, true))
    return Res;
}

if (VT == MVT::i64 &&
    DAG.ComputeNumSignBits(LHS) > 32 &&
    DAG.ComputeNumSignBits(RHS) > 32) {
  EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());

  //HiLo split
  SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
  SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
  SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
                               LHS_Lo, RHS_Lo);
  SDValue Res[2] = {
    DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
    DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
  };
  return DAG.getMergeValues(Res, DL);
}

SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
SDValue RSign = LHSign; // Remainder sign is the same as LHS

LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);

LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);

SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
SDValue Rem = Div.getValue(1);

Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);

Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);

SDValue Res[2] = {
  Div,
  Rem
};
return DAG.getMergeValues(Res, DL);
2134}

2136// (frem x, y) -> (fma (fneg (ftrunc (fdiv x, y))), y, x)
2137SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
EVT VT = Op.getValueType();
auto Flags = Op->getFlags();
SDValue X = Op.getOperand(0);
SDValue Y = Op.getOperand(1);

SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags);
SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags);
SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags);
// TODO: For f32 use FMAD instead if !hasFastFMA32?
return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags);
2149}

2151SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

// result = trunc(src)
// if (src > 0.0 && src != result)
//   result += 1.0

SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);

const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);

EVT SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);

SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);

SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
// TODO: Should this propagate fast-math-flags?
return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2174}

2176static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
                                SelectionDAG &DAG) {
const unsigned FractBits = 52;
const unsigned ExpBits = 11;

SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
                              Hi,
                              DAG.getConstant(FractBits - 32, SL, MVT::i32),
                              DAG.getConstant(ExpBits, SL, MVT::i32));
SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
                          DAG.getConstant(1023, SL, MVT::i32));

return Exp;
2189}

2191SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

assert(Op.getValueType() == MVT::f64)(static_cast <bool> (Op.getValueType() == MVT::f64) ? void
 (0) : __assert_fail ("Op.getValueType() == MVT::f64", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 2195, __extension__ __PRETTY_FUNCTION__));

const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);

// Extract the upper half, since this is where we will find the sign and
// exponent.
SDValue Hi = getHiHalf64(Src, DAG);

SDValue Exp = extractF64Exponent(Hi, SL, DAG);

const unsigned FractBits = 52;

// Extract the sign bit.
const SDValue SignBitMask = DAG.getConstant(UINT32_C(1)1U << 31, SL, MVT::i32);
SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);

// Extend back to 64-bits.
SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);

SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
const SDValue FractMask
  = DAG.getConstant((UINT64_C(1)1UL << FractBits) - 1, SL, MVT::i64);

SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);

EVT SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);

const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);

SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);

SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);

return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
2235}

2237SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

assert(Op.getValueType() == MVT::f64)(static_cast <bool> (Op.getValueType() == MVT::f64) ? void
 (0) : __assert_fail ("Op.getValueType() == MVT::f64", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 2241, __extension__ __PRETTY_FUNCTION__));

APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);

// TODO: Should this propagate fast-math-flags?

SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);

SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);

APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);

EVT SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);

return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
2262}

2264SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
// FNEARBYINT and FRINT are the same, except in their handling of FP
// exceptions. Those aren't really meaningful for us, and OpenCL only has
// rint, so just treat them as equivalent.
return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0));
2269}

2271// XXX - May require not supporting f32 denormals?

2273// Don't handle v2f16. The extra instructions to scalarize and repack around the
2274// compare and vselect end up producing worse code than scalarizing the whole
2275// operation.
2276SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue X = Op.getOperand(0);
EVT VT = Op.getValueType();

SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);

// TODO: Should this propagate fast-math-flags?

SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);

SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);

const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
const SDValue One = DAG.getConstantFP(1.0, SL, VT);
const SDValue Half = DAG.getConstantFP(0.5, SL, VT);

SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);

EVT SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);

SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);

return DAG.getNode(ISD::FADD, SL, VT, T, Sel);
2303}

2305SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

// result = trunc(src);
// if (src < 0.0 && src != result)
//   result += -1.0.

SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);

const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);

EVT SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);

SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);

SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
// TODO: Should this propagate fast-math-flags?
return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2328}

2330SDValue AMDGPUTargetLowering::LowerFLOG(SDValue Op, SelectionDAG &DAG,
                                      double Log2BaseInverted) const {
EVT VT = Op.getValueType();

SDLoc SL(Op);
SDValue Operand = Op.getOperand(0);
SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand);
SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);

return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand);
2340}

2342// exp2(M_LOG2E_F * f);
2343SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType();
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

const SDValue K = DAG.getConstantFP(numbers::log2e, SL, VT);
SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags());
return DAG.getNode(ISD::FEXP2, SL, VT, Mul, Op->getFlags());
2351}

2353static bool isCtlzOpc(unsigned Opc) {
return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
2355}

2357static bool isCttzOpc(unsigned Opc) {
return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF;
2359}

2361SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

assert(isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode()))(static_cast <bool> (isCtlzOpc(Op.getOpcode()) || isCttzOpc
(Op.getOpcode())) ? void (0) : __assert_fail ("isCtlzOpc(Op.getOpcode()) || isCttzOpc(Op.getOpcode())"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2365, __extension__
 __PRETTY_FUNCTION__));
bool Ctlz = isCtlzOpc(Op.getOpcode());
unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;

bool ZeroUndef = Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF ||
                 Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF;

if (Src.getValueType() == MVT::i32) {
  // (ctlz hi:lo) -> (umin (ffbh src), 32)
  // (cttz hi:lo) -> (umin (ffbl src), 32)
  // (ctlz_zero_undef src) -> (ffbh src)
  // (cttz_zero_undef src) -> (ffbl src)
  SDValue NewOpr = DAG.getNode(NewOpc, SL, MVT::i32, Src);
  if (!ZeroUndef) {
    const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32);
    NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const32);
  }
  return NewOpr;
}

SDValue Lo, Hi;
std::tie(Lo, Hi) = split64BitValue(Src, DAG);

SDValue OprLo = DAG.getNode(NewOpc, SL, MVT::i32, Lo);
SDValue OprHi = DAG.getNode(NewOpc, SL, MVT::i32, Hi);

// (ctlz hi:lo) -> (umin3 (ffbh hi), (uaddsat (ffbh lo), 32), 64)
// (cttz hi:lo) -> (umin3 (uaddsat (ffbl hi), 32), (ffbl lo), 64)
// (ctlz_zero_undef hi:lo) -> (umin (ffbh hi), (add (ffbh lo), 32))
// (cttz_zero_undef hi:lo) -> (umin (add (ffbl hi), 32), (ffbl lo))

unsigned AddOpc = ZeroUndef ? ISD::ADD : ISD::UADDSAT;
const SDValue Const32 = DAG.getConstant(32, SL, MVT::i32);
if (Ctlz)
  OprLo = DAG.getNode(AddOpc, SL, MVT::i32, OprLo, Const32);
else
  OprHi = DAG.getNode(AddOpc, SL, MVT::i32, OprHi, Const32);

SDValue NewOpr;
NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, OprLo, OprHi);
if (!ZeroUndef) {
  const SDValue Const64 = DAG.getConstant(64, SL, MVT::i32);
  NewOpr = DAG.getNode(ISD::UMIN, SL, MVT::i32, NewOpr, Const64);
}

return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
2411}

2413SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
                                             bool Signed) const {
// The regular method converting a 64-bit integer to float roughly consists of
// 2 steps: normalization and rounding. In fact, after normalization, the
// conversion from a 64-bit integer to a float is essentially the same as the
// one from a 32-bit integer. The only difference is that it has more
// trailing bits to be rounded. To leverage the native 32-bit conversion, a
// 64-bit integer could be preprocessed and fit into a 32-bit integer then
// converted into the correct float number. The basic steps for the unsigned
// conversion are illustrated in the following pseudo code:
//
// f32 uitofp(i64 u) {
//   i32 hi, lo = split(u);
//   // Only count the leading zeros in hi as we have native support of the
//   // conversion from i32 to f32. If hi is all 0s, the conversion is
//   // reduced to a 32-bit one automatically.
//   i32 shamt = clz(hi); // Return 32 if hi is all 0s.
//   u <<= shamt;
//   hi, lo = split(u);
//   hi |= (lo != 0) ? 1 : 0; // Adjust rounding bit in hi based on lo.
//   // convert it as a 32-bit integer and scale the result back.
//   return uitofp(hi) * 2^(32 - shamt);
// }
//
// The signed one follows the same principle but uses 'ffbh_i32' to count its
// sign bits instead. If 'ffbh_i32' is not available, its absolute value is
// converted instead followed by negation based its sign bit.

SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

SDValue Lo, Hi;
std::tie(Lo, Hi) = split64BitValue(Src, DAG);
SDValue Sign;
SDValue ShAmt;
if (Signed && Subtarget->isGCN()) {
  // We also need to consider the sign bit in Lo if Hi has just sign bits,
  // i.e. Hi is 0 or -1. However, that only needs to take the MSB into
  // account. That is, the maximal shift is
  // - 32 if Lo and Hi have opposite signs;
  // - 33 if Lo and Hi have the same sign.
  //
  // Or, MaxShAmt = 33 + OppositeSign, where
  //
  // OppositeSign is defined as ((Lo ^ Hi) >> 31), which is
  // - -1 if Lo and Hi have opposite signs; and
  // -  0 otherwise.
  //
  // All in all, ShAmt is calculated as
  //
  //  umin(sffbh(Hi), 33 + (Lo^Hi)>>31) - 1.
  //
  // or
  //
  //  umin(sffbh(Hi) - 1, 32 + (Lo^Hi)>>31).
  //
  // to reduce the critical path.
  SDValue OppositeSign = DAG.getNode(
      ISD::SRA, SL, MVT::i32, DAG.getNode(ISD::XOR, SL, MVT::i32, Lo, Hi),
      DAG.getConstant(31, SL, MVT::i32));
  SDValue MaxShAmt =
      DAG.getNode(ISD::ADD, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32),
                  OppositeSign);
  // Count the leading sign bits.
  ShAmt = DAG.getNode(AMDGPUISD::FFBH_I32, SL, MVT::i32, Hi);
  // Different from unsigned conversion, the shift should be one bit less to
  // preserve the sign bit.
  ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, ShAmt,
                      DAG.getConstant(1, SL, MVT::i32));
  ShAmt = DAG.getNode(ISD::UMIN, SL, MVT::i32, ShAmt, MaxShAmt);
} else {
  if (Signed) {
    // Without 'ffbh_i32', only leading zeros could be counted. Take the
    // absolute value first.
    Sign = DAG.getNode(ISD::SRA, SL, MVT::i64, Src,
                       DAG.getConstant(63, SL, MVT::i64));
    SDValue Abs =
        DAG.getNode(ISD::XOR, SL, MVT::i64,
                    DAG.getNode(ISD::ADD, SL, MVT::i64, Src, Sign), Sign);
    std::tie(Lo, Hi) = split64BitValue(Abs, DAG);
  }
  // Count the leading zeros.
  ShAmt = DAG.getNode(ISD::CTLZ, SL, MVT::i32, Hi);
  // The shift amount for signed integers is [0, 32].
}
// Normalize the given 64-bit integer.
SDValue Norm = DAG.getNode(ISD::SHL, SL, MVT::i64, Src, ShAmt);
// Split it again.
std::tie(Lo, Hi) = split64BitValue(Norm, DAG);
// Calculate the adjust bit for rounding.
// (lo != 0) ? 1 : 0 => (lo >= 1) ? 1 : 0 => umin(1, lo)
SDValue Adjust = DAG.getNode(ISD::UMIN, SL, MVT::i32,
                             DAG.getConstant(1, SL, MVT::i32), Lo);
// Get the 32-bit normalized integer.
Norm = DAG.getNode(ISD::OR, SL, MVT::i32, Hi, Adjust);
// Convert the normalized 32-bit integer into f32.
unsigned Opc =
    (Signed && Subtarget->isGCN()) ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
SDValue FVal = DAG.getNode(Opc, SL, MVT::f32, Norm);

// Finally, need to scale back the converted floating number as the original
// 64-bit integer is converted as a 32-bit one.
ShAmt = DAG.getNode(ISD::SUB, SL, MVT::i32, DAG.getConstant(32, SL, MVT::i32),
                    ShAmt);
// On GCN, use LDEXP directly.
if (Subtarget->isGCN())
  return DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f32, FVal, ShAmt);

// Otherwise, align 'ShAmt' to the exponent part and add it into the exponent
// part directly to emulate the multiplication of 2^ShAmt. That 8-bit
// exponent is enough to avoid overflowing into the sign bit.
SDValue Exp = DAG.getNode(ISD::SHL, SL, MVT::i32, ShAmt,
                          DAG.getConstant(23, SL, MVT::i32));
SDValue IVal =
    DAG.getNode(ISD::ADD, SL, MVT::i32,
                DAG.getNode(ISD::BITCAST, SL, MVT::i32, FVal), Exp);
if (Signed) {
  // Set the sign bit.
  Sign = DAG.getNode(ISD::SHL, SL, MVT::i32,
                     DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Sign),
                     DAG.getConstant(31, SL, MVT::i32));
  IVal = DAG.getNode(ISD::OR, SL, MVT::i32, IVal, Sign);
}
return DAG.getNode(ISD::BITCAST, SL, MVT::f32, IVal);
2537}

2539SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
                                             bool Signed) const {
SDLoc SL(Op);
SDValue Src = Op.getOperand(0);

SDValue Lo, Hi;
std::tie(Lo, Hi) = split64BitValue(Src, DAG);

SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
                            SL, MVT::f64, Hi);

SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);

SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
                            DAG.getConstant(32, SL, MVT::i32));
// TODO: Should this propagate fast-math-flags?
return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
2556}

2558SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
                                             SelectionDAG &DAG) const {
// TODO: Factor out code common with LowerSINT_TO_FP.
EVT DestVT = Op.getValueType();
SDValue Src = Op.getOperand(0);
EVT SrcVT = Src.getValueType();

if (SrcVT == MVT::i16) {
  if (DestVT == MVT::f16)
    return Op;
  SDLoc DL(Op);

  // Promote src to i32
  SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src);
  return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext);
}

assert(SrcVT == MVT::i64 && "operation should be legal")(static_cast <bool> (SrcVT == MVT::i64 && "operation should be legal"
) ? void (0) : __assert_fail ("SrcVT == MVT::i64 && \"operation should be legal\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2575, __extension__
 __PRETTY_FUNCTION__));

if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
  SDLoc DL(Op);

  SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
  SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
  SDValue FPRound =
      DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);

  return FPRound;
}

if (DestVT == MVT::f32)
  return LowerINT_TO_FP32(Op, DAG, false);

assert(DestVT == MVT::f64)(static_cast <bool> (DestVT == MVT::f64) ? void (0) : __assert_fail
 ("DestVT == MVT::f64", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 2591, __extension__ __PRETTY_FUNCTION__));
return LowerINT_TO_FP64(Op, DAG, false);
2593}

2595SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
                                            SelectionDAG &DAG) const {
EVT DestVT = Op.getValueType();

SDValue Src = Op.getOperand(0);
EVT SrcVT = Src.getValueType();

if (SrcVT == MVT::i16) {
  if (DestVT == MVT::f16)
    return Op;

  SDLoc DL(Op);
  // Promote src to i32
  SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src);
  return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext);
}

assert(SrcVT == MVT::i64 && "operation should be legal")(static_cast <bool> (SrcVT == MVT::i64 && "operation should be legal"
) ? void (0) : __assert_fail ("SrcVT == MVT::i64 && \"operation should be legal\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2612, __extension__
 __PRETTY_FUNCTION__));

// TODO: Factor out code common with LowerUINT_TO_FP.

if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
  SDLoc DL(Op);
  SDValue Src = Op.getOperand(0);

  SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
  SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
  SDValue FPRound =
      DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);

  return FPRound;
}

if (DestVT == MVT::f32)
  return LowerINT_TO_FP32(Op, DAG, true);

assert(DestVT == MVT::f64)(static_cast <bool> (DestVT == MVT::f64) ? void (0) : __assert_fail
 ("DestVT == MVT::f64", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 2631, __extension__ __PRETTY_FUNCTION__));
return LowerINT_TO_FP64(Op, DAG, true);
2633}

2635SDValue AMDGPUTargetLowering::LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG,
                                             bool Signed) const {
SDLoc SL(Op);

SDValue Src = Op.getOperand(0);
EVT SrcVT = Src.getValueType();

assert(SrcVT == MVT::f32 || SrcVT == MVT::f64)(static_cast <bool> (SrcVT == MVT::f32 || SrcVT == MVT::
f64) ? void (0) : __assert_fail ("SrcVT == MVT::f32 || SrcVT == MVT::f64"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2642, __extension__
 __PRETTY_FUNCTION__));

// The basic idea of converting a floating point number into a pair of 32-bit
// integers is illustrated as follows:
//
//     tf := trunc(val);
//    hif := floor(tf * 2^-32);
//    lof := tf - hif * 2^32; // lof is always positive due to floor.
//     hi := fptoi(hif);
//     lo := fptoi(lof);
//
SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, SrcVT, Src);
SDValue Sign;
if (Signed && SrcVT == MVT::f32) {
  // However, a 32-bit floating point number has only 23 bits mantissa and
  // it's not enough to hold all the significant bits of `lof` if val is
  // negative. To avoid the loss of precision, We need to take the absolute
  // value after truncating and flip the result back based on the original
  // signedness.
  Sign = DAG.getNode(ISD::SRA, SL, MVT::i32,
                     DAG.getNode(ISD::BITCAST, SL, MVT::i32, Trunc),
                     DAG.getConstant(31, SL, MVT::i32));
  Trunc = DAG.getNode(ISD::FABS, SL, SrcVT, Trunc);
}

SDValue K0, K1;
if (SrcVT == MVT::f64) {
  K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(/*2^-32*/ 0x3df0000000000000)0x3df0000000000000UL),
                         SL, SrcVT);
  K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(/*-2^32*/ 0xc1f0000000000000)0xc1f0000000000000UL),
                         SL, SrcVT);
} else {
  K0 = DAG.getConstantFP(BitsToFloat(UINT32_C(/*2^-32*/ 0x2f800000)0x2f800000U), SL,
                         SrcVT);
  K1 = DAG.getConstantFP(BitsToFloat(UINT32_C(/*-2^32*/ 0xcf800000)0xcf800000U), SL,
                         SrcVT);
}
// TODO: Should this propagate fast-math-flags?
SDValue Mul = DAG.getNode(ISD::FMUL, SL, SrcVT, Trunc, K0);

SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, SrcVT, Mul);

SDValue Fma = DAG.getNode(ISD::FMA, SL, SrcVT, FloorMul, K1, Trunc);

SDValue Hi = DAG.getNode((Signed && SrcVT == MVT::f64) ? ISD::FP_TO_SINT
                                                       : ISD::FP_TO_UINT,
                         SL, MVT::i32, FloorMul);
SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);

SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64,
                             DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi}));

if (Signed && SrcVT == MVT::f32) {
  assert(Sign)(static_cast <bool> (Sign) ? void (0) : __assert_fail (
"Sign", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2695
, __extension__ __PRETTY_FUNCTION__));
  // Flip the result based on the signedness, which is either all 0s or 1s.
  Sign = DAG.getNode(ISD::BITCAST, SL, MVT::i64,
                     DAG.getBuildVector(MVT::v2i32, SL, {Sign, Sign}));
  // r := xor(r, sign) - sign;
  Result =
      DAG.getNode(ISD::SUB, SL, MVT::i64,
                  DAG.getNode(ISD::XOR, SL, MVT::i64, Result, Sign), Sign);
}

return Result;
2706}

2708SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
SDLoc DL(Op);
SDValue N0 = Op.getOperand(0);

// Convert to target node to get known bits
if (N0.getValueType() == MVT::f32)
  return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);

if (getTargetMachine().Options.UnsafeFPMath) {
  // There is a generic expand for FP_TO_FP16 with unsafe fast math.
  return SDValue();
}

assert(N0.getSimpleValueType() == MVT::f64)(static_cast <bool> (N0.getSimpleValueType() == MVT::f64
) ? void (0) : __assert_fail ("N0.getSimpleValueType() == MVT::f64"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2721, __extension__
 __PRETTY_FUNCTION__));

// f64 -> f16 conversion using round-to-nearest-even rounding mode.
const unsigned ExpMask = 0x7ff;
const unsigned ExpBiasf64 = 1023;
const unsigned ExpBiasf16 = 15;
SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
SDValue One = DAG.getConstant(1, DL, MVT::i32);
SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
                         DAG.getConstant(32, DL, MVT::i64));
UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
                        DAG.getConstant(20, DL, MVT::i64));
E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
                DAG.getConstant(ExpMask, DL, MVT::i32));
// Subtract the fp64 exponent bias (1023) to get the real exponent and
// add the f16 bias (15) to get the biased exponent for the f16 format.
E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
                DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));

SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
                        DAG.getConstant(8, DL, MVT::i32));
M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
                DAG.getConstant(0xffe, DL, MVT::i32));

SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
                                DAG.getConstant(0x1ff, DL, MVT::i32));
MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);

SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);

// (M != 0 ? 0x0200 : 0) | 0x7c00;
SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
    DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
                    Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));

// N = M | (E << 12);
SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
    DAG.getNode(ISD::SHL, DL, MVT::i32, E,
                DAG.getConstant(12, DL, MVT::i32)));

// B = clamp(1-E, 0, 13);
SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
                                One, E);
SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
                DAG.getConstant(13, DL, MVT::i32));

SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
                                 DAG.getConstant(0x1000, DL, MVT::i32));

SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);

SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
                            DAG.getConstant(0x7, DL, MVT::i32));
V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
                DAG.getConstant(2, DL, MVT::i32));
SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
                             One, Zero, ISD::SETEQ);
SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
                             One, Zero, ISD::SETGT);
V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);

V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
                    DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
                    I, V, ISD::SETEQ);

// Extract the sign bit.
SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
                          DAG.getConstant(16, DL, MVT::i32));
Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
                   DAG.getConstant(0x8000, DL, MVT::i32));

V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
2805}

2807SDValue AMDGPUTargetLowering::LowerFP_TO_INT(SDValue Op,
                                           SelectionDAG &DAG) const {
SDValue Src = Op.getOperand(0);
unsigned OpOpcode = Op.getOpcode();
EVT SrcVT = Src.getValueType();
EVT DestVT = Op.getValueType();

// Will be selected natively
if (SrcVT == MVT::f16 && DestVT == MVT::i16)
  return Op;

// Promote i16 to i32
if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
  SDLoc DL(Op);

  SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src);
  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToInt32);
}

if (SrcVT == MVT::f16 ||
    (SrcVT == MVT::f32 && Src.getOpcode() == ISD::FP16_TO_FP)) {
  SDLoc DL(Op);

  SDValue FpToInt32 = DAG.getNode(OpOpcode, DL, MVT::i32, Src);
  unsigned Ext =
      OpOpcode == ISD::FP_TO_SINT ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
  return DAG.getNode(Ext, DL, MVT::i64, FpToInt32);
}

if (DestVT == MVT::i64 && (SrcVT == MVT::f32 || SrcVT == MVT::f64))
  return LowerFP_TO_INT64(Op, DAG, OpOpcode == ISD::FP_TO_SINT);

return SDValue();
2840}

2842SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
                                                   SelectionDAG &DAG) const {
EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
MVT VT = Op.getSimpleValueType();
MVT ScalarVT = VT.getScalarType();

assert(VT.isVector())(static_cast <bool> (VT.isVector()) ? void (0) : __assert_fail
 ("VT.isVector()", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 2848, __extension__ __PRETTY_FUNCTION__));

SDValue Src = Op.getOperand(0);
SDLoc DL(Op);

// TODO: Don't scalarize on Evergreen?
unsigned NElts = VT.getVectorNumElements();
SmallVector<SDValue, 8> Args;
DAG.ExtractVectorElements(Src, Args, 0, NElts);

SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
for (unsigned I = 0; I < NElts; ++I)
  Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);

return DAG.getBuildVector(VT, DL, Args);
2863}

2865//===----------------------------------------------------------------------===//
2866// Custom DAG optimizations
2867//===----------------------------------------------------------------------===//

2869static bool isU24(SDValue Op, SelectionDAG &DAG) {
return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
2871}

2873static bool isI24(SDValue Op, SelectionDAG &DAG) {
EVT VT = Op.getValueType();
return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
                                   // as unsigned 24-bit values.
       AMDGPUTargetLowering::numBitsSigned(Op, DAG) <= 24;
2878}

2880static SDValue simplifyMul24(SDNode *Node24,
                           TargetLowering::DAGCombinerInfo &DCI) {
SelectionDAG &DAG = DCI.DAG;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;

SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0);
SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1);
unsigned NewOpcode = Node24->getOpcode();
if (IsIntrin) {
  unsigned IID = cast<ConstantSDNode>(Node24->getOperand(0))->getZExtValue();
  switch (IID) {
  case Intrinsic::amdgcn_mul_i24:
    NewOpcode = AMDGPUISD::MUL_I24;
    break;
  case Intrinsic::amdgcn_mul_u24:
    NewOpcode = AMDGPUISD::MUL_U24;
    break;
  case Intrinsic::amdgcn_mulhi_i24:
    NewOpcode = AMDGPUISD::MULHI_I24;
    break;
  case Intrinsic::amdgcn_mulhi_u24:
    NewOpcode = AMDGPUISD::MULHI_U24;
    break;
  default:
    llvm_unreachable("Expected 24-bit mul intrinsic")::llvm::llvm_unreachable_internal("Expected 24-bit mul intrinsic"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 2905);
  }
}

APInt Demanded = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 24);

// First try to simplify using SimplifyMultipleUseDemandedBits which allows
// the operands to have other uses, but will only perform simplifications that
// involve bypassing some nodes for this user.
SDValue DemandedLHS = TLI.SimplifyMultipleUseDemandedBits(LHS, Demanded, DAG);
SDValue DemandedRHS = TLI.SimplifyMultipleUseDemandedBits(RHS, Demanded, DAG);
if (DemandedLHS || DemandedRHS)
  return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(),
                     DemandedLHS ? DemandedLHS : LHS,
                     DemandedRHS ? DemandedRHS : RHS);

// Now try SimplifyDemandedBits which can simplify the nodes used by our
// operands if this node is the only user.
if (TLI.SimplifyDemandedBits(LHS, Demanded, DCI))
  return SDValue(Node24, 0);
if (TLI.SimplifyDemandedBits(RHS, Demanded, DCI))
  return SDValue(Node24, 0);

return SDValue();
2929}

2931template <typename IntTy>
2932static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
                             uint32_t Width, const SDLoc &DL) {
if (Width + Offset < 32) {
  uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
  IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
  return DAG.getConstant(Result, DL, MVT::i32);
}

return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
2941}

2943static bool hasVolatileUser(SDNode *Val) {
for (SDNode *U : Val->uses()) {
  if (MemSDNode *M = dyn_cast<MemSDNode>(U)) {
    if (M->isVolatile())
      return true;
  }
}

return false;
2952}

2954bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
// i32 vectors are the canonical memory type.
if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
  return false;

if (!VT.isByteSized())
  return false;

unsigned Size = VT.getStoreSize();

if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
  return false;

if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
  return false;

return true;
2971}

2973// Replace load of an illegal type with a store of a bitcast to a friendlier
2974// type.
2975SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
                                               DAGCombinerInfo &DCI) const {
if (!DCI.isBeforeLegalize())
  return SDValue();

LoadSDNode *LN = cast<LoadSDNode>(N);
if (!LN->isSimple() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN))
  return SDValue();

SDLoc SL(N);
SelectionDAG &DAG = DCI.DAG;
EVT VT = LN->getMemoryVT();

unsigned Size = VT.getStoreSize();
Align Alignment = LN->getAlign();
if (Alignment < Size && isTypeLegal(VT)) {
  bool IsFast;
  unsigned AS = LN->getAddressSpace();

  // Expand unaligned loads earlier than legalization. Due to visitation order
  // problems during legalization, the emitted instructions to pack and unpack
  // the bytes again are not eliminated in the case of an unaligned copy.
  if (!allowsMisalignedMemoryAccesses(
          VT, AS, Alignment, LN->getMemOperand()->getFlags(), &IsFast)) {
    if (VT.isVector())
      return SplitVectorLoad(SDValue(LN, 0), DAG);

    SDValue Ops[2];
    std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);

    return DAG.getMergeValues(Ops, SDLoc(N));
  }

  if (!IsFast)
    return SDValue();
}

if (!shouldCombineMemoryType(VT))
  return SDValue();

EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);

SDValue NewLoad
  = DAG.getLoad(NewVT, SL, LN->getChain(),
                LN->getBasePtr(), LN->getMemOperand());

SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
DCI.CombineTo(N, BC, NewLoad.getValue(1));
return SDValue(N, 0);
3024}

3026// Replace store of an illegal type with a store of a bitcast to a friendlier
3027// type.
3028SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
if (!DCI.isBeforeLegalize())
  return SDValue();

StoreSDNode *SN = cast<StoreSDNode>(N);
if (!SN->isSimple() || !ISD::isNormalStore(SN))
  return SDValue();

EVT VT = SN->getMemoryVT();
unsigned Size = VT.getStoreSize();

SDLoc SL(N);
SelectionDAG &DAG = DCI.DAG;
Align Alignment = SN->getAlign();
if (Alignment < Size && isTypeLegal(VT)) {
  bool IsFast;
  unsigned AS = SN->getAddressSpace();

  // Expand unaligned stores earlier than legalization. Due to visitation
  // order problems during legalization, the emitted instructions to pack and
  // unpack the bytes again are not eliminated in the case of an unaligned
  // copy.
  if (!allowsMisalignedMemoryAccesses(
          VT, AS, Alignment, SN->getMemOperand()->getFlags(), &IsFast)) {
    if (VT.isVector())
      return SplitVectorStore(SDValue(SN, 0), DAG);

    return expandUnalignedStore(SN, DAG);
  }

  if (!IsFast)
    return SDValue();
}

if (!shouldCombineMemoryType(VT))
  return SDValue();

EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
SDValue Val = SN->getValue();

//DCI.AddToWorklist(Val.getNode());

bool OtherUses = !Val.hasOneUse();
SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
if (OtherUses) {
  SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
  DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
}

return DAG.getStore(SN->getChain(), SL, CastVal,
                    SN->getBasePtr(), SN->getMemOperand());
3080}

3082// FIXME: This should go in generic DAG combiner with an isTruncateFree check,
3083// but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
3084// issues.
3085SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
                                                      DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
SDValue N0 = N->getOperand(0);

// (vt2 (assertzext (truncate vt0:x), vt1)) ->
//     (vt2 (truncate (assertzext vt0:x, vt1)))
if (N0.getOpcode() == ISD::TRUNCATE) {
  SDValue N1 = N->getOperand(1);
  EVT ExtVT = cast<VTSDNode>(N1)->getVT();
  SDLoc SL(N);

  SDValue Src = N0.getOperand(0);
  EVT SrcVT = Src.getValueType();
  if (SrcVT.bitsGE(ExtVT)) {
    SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
    return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg);
  }
}

return SDValue();
3106}

3108SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
SDNode *N, DAGCombinerInfo &DCI) const {
unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
switch (IID) {
case Intrinsic::amdgcn_mul_i24:
case Intrinsic::amdgcn_mul_u24:
case Intrinsic::amdgcn_mulhi_i24:
case Intrinsic::amdgcn_mulhi_u24:
  return simplifyMul24(N, DCI);
case Intrinsic::amdgcn_fract:
case Intrinsic::amdgcn_rsq:
case Intrinsic::amdgcn_rcp_legacy:
case Intrinsic::amdgcn_rsq_legacy:
case Intrinsic::amdgcn_rsq_clamp:
case Intrinsic::amdgcn_ldexp: {
  // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
  SDValue Src = N->getOperand(1);
  return Src.isUndef() ? Src : SDValue();
}
default:
  return SDValue();
}
3130}

3132/// Split the 64-bit value \p LHS into two 32-bit components, and perform the
3133/// binary operation \p Opc to it with the corresponding constant operands.
3134SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
DAGCombinerInfo &DCI, const SDLoc &SL,
unsigned Opc, SDValue LHS,
uint32_t ValLo, uint32_t ValHi) const {
SelectionDAG &DAG = DCI.DAG;
SDValue Lo, Hi;
std::tie(Lo, Hi) = split64BitValue(LHS, DAG);

SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);

SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);

// Re-visit the ands. It's possible we eliminated one of them and it could
// simplify the vector.
DCI.AddToWorklist(Lo.getNode());
DCI.AddToWorklist(Hi.getNode());

SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
3155}

3157SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);

ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
if (!RHS)
  return SDValue();

SDValue LHS = N->getOperand(0);
unsigned RHSVal = RHS->getZExtValue();
if (!RHSVal)
  return LHS;

SDLoc SL(N);
SelectionDAG &DAG = DCI.DAG;

switch (LHS->getOpcode()) {
default:
  break;
case ISD::ZERO_EXTEND:
case ISD::SIGN_EXTEND:
case ISD::ANY_EXTEND: {
  SDValue X = LHS->getOperand(0);

  if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
      isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) {
    // Prefer build_vector as the canonical form if packed types are legal.
    // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
    SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
     { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) });
    return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
  }

  // shl (ext x) => zext (shl x), if shift does not overflow int
  if (VT != MVT::i64)
    break;
  KnownBits Known = DAG.computeKnownBits(X);
  unsigned LZ = Known.countMinLeadingZeros();
  if (LZ < RHSVal)
    break;
  EVT XVT = X.getValueType();
  SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
  return DAG.getZExtOrTrunc(Shl, SL, VT);
}
}

if (VT != MVT::i64)
  return SDValue();

// i64 (shl x, C) -> (build_pair 0, (shl x, C -32))

// On some subtargets, 64-bit shift is a quarter rate instruction. In the
// common case, splitting this into a move and a 32-bit shift is faster and
// the same code size.
if (RHSVal < 32)
  return SDValue();

SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);

SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);

const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);

SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
3223}

3225SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
if (N->getValueType(0) != MVT::i64)
  return SDValue();

const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
if (!RHS)
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
SDLoc SL(N);
unsigned RHSVal = RHS->getZExtValue();

// (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31)
if (RHSVal == 32) {
  SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
  SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
                                 DAG.getConstant(31, SL, MVT::i32));

  SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
}

// (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31)
if (RHSVal == 63) {
  SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
  SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
                                 DAG.getConstant(31, SL, MVT::i32));
  SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
  return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
}

return SDValue();
3258}

3260SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
if (!RHS)
  return SDValue();

EVT VT = N->getValueType(0);
SDValue LHS = N->getOperand(0);
unsigned ShiftAmt = RHS->getZExtValue();
SelectionDAG &DAG = DCI.DAG;
SDLoc SL(N);

// fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
// this improves the ability to match BFE patterns in isel.
if (LHS.getOpcode() == ISD::AND) {
  if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
    unsigned MaskIdx, MaskLen;
    if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
        MaskIdx == ShiftAmt) {
      return DAG.getNode(
          ISD::AND, SL, VT,
          DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
          DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
    }
  }
}

if (VT != MVT::i64)
  return SDValue();

if (ShiftAmt < 32)
  return SDValue();

// srl i64:x, C for C >= 32
// =>
//   build_pair (srl hi_32(x), C - 32), 0
SDValue Zero = DAG.getConstant(0, SL, MVT::i32);

SDValue Hi = getHiHalf64(LHS, DAG);

SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);

SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});

return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
3306}

3308SDValue AMDGPUTargetLowering::performTruncateCombine(
SDNode *N, DAGCombinerInfo &DCI) const {
SDLoc SL(N);
SelectionDAG &DAG = DCI.DAG;
EVT VT = N->getValueType(0);
SDValue Src = N->getOperand(0);

// vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
  SDValue Vec = Src.getOperand(0);
  if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
    SDValue Elt0 = Vec.getOperand(0);
    EVT EltVT = Elt0.getValueType();
    if (VT.getFixedSizeInBits() <= EltVT.getFixedSizeInBits()) {
      if (EltVT.isFloatingPoint()) {
        Elt0 = DAG.getNode(ISD::BITCAST, SL,
                           EltVT.changeTypeToInteger(), Elt0);
      }

      return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
    }
  }
}

// Equivalent of above for accessing the high element of a vector as an
// integer operation.
// trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
  if (auto K = isConstOrConstSplat(Src.getOperand(1))) {
    if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
      SDValue BV = stripBitcast(Src.getOperand(0));
      if (BV.getOpcode() == ISD::BUILD_VECTOR &&
          BV.getValueType().getVectorNumElements() == 2) {
        SDValue SrcElt = BV.getOperand(1);
        EVT SrcEltVT = SrcElt.getValueType();
        if (SrcEltVT.isFloatingPoint()) {
          SrcElt = DAG.getNode(ISD::BITCAST, SL,
                               SrcEltVT.changeTypeToInteger(), SrcElt);
        }

        return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
      }
    }
  }
}

// Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
//
// i16 (trunc (srl i64:x, K)), K <= 16 ->
//     i16 (trunc (srl (i32 (trunc x), K)))
if (VT.getScalarSizeInBits() < 32) {
  EVT SrcVT = Src.getValueType();
  if (SrcVT.getScalarSizeInBits() > 32 &&
      (Src.getOpcode() == ISD::SRL ||
       Src.getOpcode() == ISD::SRA ||
       Src.getOpcode() == ISD::SHL)) {
    SDValue Amt = Src.getOperand(1);
    KnownBits Known = DAG.computeKnownBits(Amt);
    unsigned Size = VT.getScalarSizeInBits();
    if ((Known.isConstant() && Known.getConstant().ule(Size)) ||
        (Known.countMaxActiveBits() <= Log2_32(Size))) {
      EVT MidVT = VT.isVector() ?
        EVT::getVectorVT(*DAG.getContext(), MVT::i32,
                         VT.getVectorNumElements()) : MVT::i32;

      EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout());
      SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
                                  Src.getOperand(0));
      DCI.AddToWorklist(Trunc.getNode());

      if (Amt.getValueType() != NewShiftVT) {
        Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT);
        DCI.AddToWorklist(Amt.getNode());
      }

      SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
                                        Trunc, Amt);
      return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
    }
  }
}

return SDValue();
3391}

3393// We need to specifically handle i64 mul here to avoid unnecessary conversion
3394// instructions. If we only match on the legalized i64 mul expansion,
3395// SimplifyDemandedBits will be unable to remove them because there will be
3396// multiple uses due to the separate mul + mulh[su].
3397static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
                      SDValue N0, SDValue N1, unsigned Size, bool Signed) {
if (Size <= 32) {
  unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
  return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
}

unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;

SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);

return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, MulLo, MulHi);
3411}

3413SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);

// Don't generate 24-bit multiplies on values that are in SGPRs, since
// we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
// unnecessarily). isDivergent() is used as an approximation of whether the
// value is in an SGPR.
if (!N->isDivergent())
  return SDValue();

unsigned Size = VT.getSizeInBits();
if (VT.isVector() || Size > 64)
  return SDValue();

// There are i16 integer mul/mad.
if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

// SimplifyDemandedBits has the annoying habit of turning useful zero_extends
// in the source into any_extends if the result of the mul is truncated. Since
// we can assume the high bits are whatever we want, use the underlying value
// to avoid the unknown high bits from interfering.
if (N0.getOpcode() == ISD::ANY_EXTEND)
  N0 = N0.getOperand(0);

if (N1.getOpcode() == ISD::ANY_EXTEND)
  N1 = N1.getOperand(0);

SDValue Mul;

if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
  N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
  N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
  Mul = getMul24(DAG, DL, N0, N1, Size, false);
} else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
  N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
  N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
  Mul = getMul24(DAG, DL, N0, N1, Size, true);
} else {
  return SDValue();
}

// We need to use sext even for MUL_U24, because MUL_U24 is used
// for signed multiply of 8 and 16-bit types.
return DAG.getSExtOrTrunc(Mul, DL, VT);
3465}

3467SDValue
3468AMDGPUTargetLowering::performMulLoHiCombine(SDNode *N,
                                          DAGCombinerInfo &DCI) const {
if (N->getValueType(0) != MVT::i32)
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

// SimplifyDemandedBits has the annoying habit of turning useful zero_extends
// in the source into any_extends if the result of the mul is truncated. Since
// we can assume the high bits are whatever we want, use the underlying value
// to avoid the unknown high bits from interfering.
if (N0.getOpcode() == ISD::ANY_EXTEND)
  N0 = N0.getOperand(0);
if (N1.getOpcode() == ISD::ANY_EXTEND)
  N1 = N1.getOperand(0);

// Try to use two fast 24-bit multiplies (one for each half of the result)
// instead of one slow extending multiply.
unsigned LoOpcode, HiOpcode;
if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
  N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
  N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
  LoOpcode = AMDGPUISD::MUL_U24;
  HiOpcode = AMDGPUISD::MULHI_U24;
} else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
  N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
  N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
  LoOpcode = AMDGPUISD::MUL_I24;
  HiOpcode = AMDGPUISD::MULHI_I24;
} else {
  return SDValue();
}

SDValue Lo = DAG.getNode(LoOpcode, DL, MVT::i32, N0, N1);
SDValue Hi = DAG.getNode(HiOpcode, DL, MVT::i32, N0, N1);
DCI.CombineTo(N, Lo, Hi);
return SDValue(N, 0);
3509}

3511SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);

if (!Subtarget->hasMulI24() || VT.isVector())
  return SDValue();

// Don't generate 24-bit multiplies on values that are in SGPRs, since
// we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
// unnecessarily). isDivergent() is used as an approximation of whether the
// value is in an SGPR.
// This doesn't apply if no s_mul_hi is available (since we'll end up with a
// valu op anyway)
if (Subtarget->hasSMulHi() && !N->isDivergent())
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

if (!isI24(N0, DAG) || !isI24(N1, DAG))
  return SDValue();

N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);

SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
DCI.AddToWorklist(Mulhi.getNode());
return DAG.getSExtOrTrunc(Mulhi, DL, VT);
3542}

3544SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);

if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
  return SDValue();

// Don't generate 24-bit multiplies on values that are in SGPRs, since
// we only have a 32-bit scalar multiply (avoid values being moved to VGPRs
// unnecessarily). isDivergent() is used as an approximation of whether the
// value is in an SGPR.
// This doesn't apply if no s_mul_hi is available (since we'll end up with a
// valu op anyway)
if (Subtarget->hasSMulHi() && !N->isDivergent())
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

if (!isU24(N0, DAG) || !isU24(N1, DAG))
  return SDValue();

N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);

SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
DCI.AddToWorklist(Mulhi.getNode());
return DAG.getZExtOrTrunc(Mulhi, DL, VT);
3575}

3577static bool isNegativeOne(SDValue Val) {
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
  return C->isAllOnes();
return false;
3581}

3583SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
                                        SDValue Op,
                                        const SDLoc &DL,
                                        unsigned Opc) const {
EVT VT = Op.getValueType();
EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() &&
                            LegalVT != MVT::i16))
  return SDValue();

if (VT != MVT::i32)
  Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);

SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
if (VT != MVT::i32)
  FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);

return FFBX;
3601}

3603// The native instructions return -1 on 0 input. Optimize out a select that
3604// produces -1 on 0.
3605//
3606// TODO: If zero is not undef, we could also do this if the output is compared
3607// against the bitwidth.
3608//
3609// TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
3610SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
                                               SDValue LHS, SDValue RHS,
                                               DAGCombinerInfo &DCI) const {
ConstantSDNode *CmpRhs = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
if (!CmpRhs || !CmpRhs->isZero())
  return SDValue();

SelectionDAG &DAG = DCI.DAG;
ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
SDValue CmpLHS = Cond.getOperand(0);

// select (setcc x, 0, eq), -1, (ctlz_zero_undef x) -> ffbh_u32 x
// select (setcc x, 0, eq), -1, (cttz_zero_undef x) -> ffbl_u32 x
if (CCOpcode == ISD::SETEQ &&
    (isCtlzOpc(RHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
    RHS.getOperand(0) == CmpLHS && isNegativeOne(LHS)) {
  unsigned Opc =
      isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
  return getFFBX_U32(DAG, CmpLHS, SL, Opc);
}

// select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x
// select (setcc x, 0, ne), (cttz_zero_undef x), -1 -> ffbl_u32 x
if (CCOpcode == ISD::SETNE &&
    (isCtlzOpc(LHS.getOpcode()) || isCttzOpc(LHS.getOpcode())) &&
    LHS.getOperand(0) == CmpLHS && isNegativeOne(RHS)) {
  unsigned Opc =
      isCttzOpc(LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;

  return getFFBX_U32(DAG, CmpLHS, SL, Opc);
}

return SDValue();
3643}

3645static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
                                       unsigned Op,
                                       const SDLoc &SL,
                                       SDValue Cond,
                                       SDValue N1,
                                       SDValue N2) {
SelectionDAG &DAG = DCI.DAG;
EVT VT = N1.getValueType();

SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
                                N1.getOperand(0), N2.getOperand(0));
DCI.AddToWorklist(NewSelect.getNode());
return DAG.getNode(Op, SL, VT, NewSelect);
3658}

3660// Pull a free FP operation out of a select so it may fold into uses.
3661//
3662// select c, (fneg x), (fneg y) -> fneg (select c, x, y)
3663// select c, (fneg x), k -> fneg (select c, x, (fneg k))
3664//
3665// select c, (fabs x), (fabs y) -> fabs (select c, x, y)
3666// select c, (fabs x), +k -> fabs (select c, x, k)
3667static SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
                                  SDValue N) {
SelectionDAG &DAG = DCI.DAG;
SDValue Cond = N.getOperand(0);
SDValue LHS = N.getOperand(1);
SDValue RHS = N.getOperand(2);

EVT VT = N.getValueType();
if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
    (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
  return distributeOpThroughSelect(DCI, LHS.getOpcode(),
                                   SDLoc(N), Cond, LHS, RHS);
}

bool Inv = false;
if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
  std::swap(LHS, RHS);
  Inv = true;
}

// TODO: Support vector constants.
ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS);
if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS) {
  SDLoc SL(N);
  // If one side is an fneg/fabs and the other is a constant, we can push the
  // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
  SDValue NewLHS = LHS.getOperand(0);
  SDValue NewRHS = RHS;

  // Careful: if the neg can be folded up, don't try to pull it back down.
  bool ShouldFoldNeg = true;

  if (NewLHS.hasOneUse()) {
    unsigned Opc = NewLHS.getOpcode();
    if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(Opc))
      ShouldFoldNeg = false;
    if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
      ShouldFoldNeg = false;
  }

  if (ShouldFoldNeg) {
    if (LHS.getOpcode() == ISD::FNEG)
      NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
    else if (CRHS->isNegative())
      return SDValue();

    if (Inv)
      std::swap(NewLHS, NewRHS);

    SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
                                    Cond, NewLHS, NewRHS);
    DCI.AddToWorklist(NewSelect.getNode());
    return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
  }
}

return SDValue();
3724}


3727SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
                                                 DAGCombinerInfo &DCI) const {
if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
  return Folded;

SDValue Cond = N->getOperand(0);
if (Cond.getOpcode() != ISD::SETCC)
  return SDValue();

EVT VT = N->getValueType(0);
SDValue LHS = Cond.getOperand(0);
SDValue RHS = Cond.getOperand(1);
SDValue CC = Cond.getOperand(2);

SDValue True = N->getOperand(1);
SDValue False = N->getOperand(2);

if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
  SelectionDAG &DAG = DCI.DAG;
  if (DAG.isConstantValueOfAnyType(True) &&
      !DAG.isConstantValueOfAnyType(False)) {
    // Swap cmp + select pair to move constant to false input.
    // This will allow using VOPC cndmasks more often.
    // select (setcc x, y), k, x -> select (setccinv x, y), x, k

    SDLoc SL(N);
    ISD::CondCode NewCC =
        getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), LHS.getValueType());

    SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
    return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
  }

  if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
    SDValue MinMax
      = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
    // Revisit this node so we can catch min3/max3/med3 patterns.
    //DCI.AddToWorklist(MinMax.getNode());
    return MinMax;
  }
}

// There's no reason to not do this if the condition has other uses.
return performCtlz_CttzCombine(SDLoc(N), Cond, True, False, DCI);
3771}

3773static bool isInv2Pi(const APFloat &APF) {
static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));

return APF.bitwiseIsEqual(KF16) ||
       APF.bitwiseIsEqual(KF32) ||
       APF.bitwiseIsEqual(KF64);
3781}

3783// 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
3784// additional cost to negate them.
3785bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N)) {
  if (C->isZero() && !C->isNegative())
    return true;

  if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(C->getValueAPF()))
    return true;
}

return false;
3795}

3797static unsigned inverseMinMax(unsigned Opc) {
switch (Opc) {
case ISD::FMAXNUM:
  return ISD::FMINNUM;
case ISD::FMINNUM:
  return ISD::FMAXNUM;
case ISD::FMAXNUM_IEEE:
  return ISD::FMINNUM_IEEE;
case ISD::FMINNUM_IEEE:
  return ISD::FMAXNUM_IEEE;
case AMDGPUISD::FMAX_LEGACY:
  return AMDGPUISD::FMIN_LEGACY;
case AMDGPUISD::FMIN_LEGACY:
  return  AMDGPUISD::FMAX_LEGACY;
default:
  llvm_unreachable("invalid min/max opcode")::llvm::llvm_unreachable_internal("invalid min/max opcode", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 3812);
}
3814}

3816SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
                                               DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
SDValue N0 = N->getOperand(0);
EVT VT = N->getValueType(0);

unsigned Opc = N0.getOpcode();

// If the input has multiple uses and we can either fold the negate down, or
// the other uses cannot, give up. This both prevents unprofitable
// transformations and infinite loops: we won't repeatedly try to fold around
// a negate that has no 'good' form.
if (N0.hasOneUse()) {
  // This may be able to fold into the source, but at a code size cost. Don't
  // fold if the fold into the user is free.
  if (allUsesHaveSourceMods(N, 0))
    return SDValue();
} else {
  if (fnegFoldsIntoOp(Opc) &&
      (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N0.getNode())))
    return SDValue();
}

SDLoc SL(N);
switch (Opc) {
case ISD::FADD: {
  if (!mayIgnoreSignedZero(N0))
    return SDValue();

  // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
  SDValue LHS = N0.getOperand(0);
  SDValue RHS = N0.getOperand(1);

  if (LHS.getOpcode() != ISD::FNEG)
    LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
  else
    LHS = LHS.getOperand(0);

  if (RHS.getOpcode() != ISD::FNEG)
    RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
  else
    RHS = RHS.getOperand(0);

  SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
  if (Res.getOpcode() != ISD::FADD)
    return SDValue(); // Op got folded away.
  if (!N0.hasOneUse())
    DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
  return Res;
}
case ISD::FMUL:
case AMDGPUISD::FMUL_LEGACY: {
  // (fneg (fmul x, y)) -> (fmul x, (fneg y))
  // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
  SDValue LHS = N0.getOperand(0);
  SDValue RHS = N0.getOperand(1);

  if (LHS.getOpcode() == ISD::FNEG)
    LHS = LHS.getOperand(0);
  else if (RHS.getOpcode() == ISD::FNEG)
    RHS = RHS.getOperand(0);
  else
    RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);

  SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
  if (Res.getOpcode() != Opc)
    return SDValue(); // Op got folded away.
  if (!N0.hasOneUse())
    DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
  return Res;
}
case ISD::FMA:
case ISD::FMAD: {
  // TODO: handle llvm.amdgcn.fma.legacy
  if (!mayIgnoreSignedZero(N0))
    return SDValue();

  // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
  SDValue LHS = N0.getOperand(0);
  SDValue MHS = N0.getOperand(1);
  SDValue RHS = N0.getOperand(2);

  if (LHS.getOpcode() == ISD::FNEG)
    LHS = LHS.getOperand(0);
  else if (MHS.getOpcode() == ISD::FNEG)
    MHS = MHS.getOperand(0);
  else
    MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);

  if (RHS.getOpcode() != ISD::FNEG)
    RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
  else
    RHS = RHS.getOperand(0);

  SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
  if (Res.getOpcode() != Opc)
    return SDValue(); // Op got folded away.
  if (!N0.hasOneUse())
    DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
  return Res;
}
case ISD::FMAXNUM:
case ISD::FMINNUM:
case ISD::FMAXNUM_IEEE:
case ISD::FMINNUM_IEEE:
case AMDGPUISD::FMAX_LEGACY:
case AMDGPUISD::FMIN_LEGACY: {
  // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
  // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
  // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
  // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)

  SDValue LHS = N0.getOperand(0);
  SDValue RHS = N0.getOperand(1);

  // 0 doesn't have a negated inline immediate.
  // TODO: This constant check should be generalized to other operations.
  if (isConstantCostlierToNegate(RHS))
    return SDValue();

  SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
  SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
  unsigned Opposite = inverseMinMax(Opc);

  SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
  if (Res.getOpcode() != Opposite)
    return SDValue(); // Op got folded away.
  if (!N0.hasOneUse())
    DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
  return Res;
}
case AMDGPUISD::FMED3: {
  SDValue Ops[3];
  for (unsigned I = 0; I < 3; ++I)
    Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());

  SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
  if (Res.getOpcode() != AMDGPUISD::FMED3)
    return SDValue(); // Op got folded away.

  if (!N0.hasOneUse()) {
    SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Res);
    DAG.ReplaceAllUsesWith(N0, Neg);

    for (SDNode *U : Neg->uses())
      DCI.AddToWorklist(U);
  }

  return Res;
}
case ISD::FP_EXTEND:
case ISD::FTRUNC:
case ISD::FRINT:
case ISD::FNEARBYINT: // XXX - Should fround be handled?
case ISD::FSIN:
case ISD::FCANONICALIZE:
case AMDGPUISD::RCP:
case AMDGPUISD::RCP_LEGACY:
case AMDGPUISD::RCP_IFLAG:
case AMDGPUISD::SIN_HW: {
  SDValue CvtSrc = N0.getOperand(0);
  if (CvtSrc.getOpcode() == ISD::FNEG) {
    // (fneg (fp_extend (fneg x))) -> (fp_extend x)
    // (fneg (rcp (fneg x))) -> (rcp x)
    return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
  }

  if (!N0.hasOneUse())
    return SDValue();

  // (fneg (fp_extend x)) -> (fp_extend (fneg x))
  // (fneg (rcp x)) -> (rcp (fneg x))
  SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
  return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
}
case ISD::FP_ROUND: {
  SDValue CvtSrc = N0.getOperand(0);

  if (CvtSrc.getOpcode() == ISD::FNEG) {
    // (fneg (fp_round (fneg x))) -> (fp_round x)
    return DAG.getNode(ISD::FP_ROUND, SL, VT,
                       CvtSrc.getOperand(0), N0.getOperand(1));
  }

  if (!N0.hasOneUse())
    return SDValue();

  // (fneg (fp_round x)) -> (fp_round (fneg x))
  SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
  return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
}
case ISD::FP16_TO_FP: {
  // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
  // f16, but legalization of f16 fneg ends up pulling it out of the source.
  // Put the fneg back as a legal source operation that can be matched later.
  SDLoc SL(N);

  SDValue Src = N0.getOperand(0);
  EVT SrcVT = Src.getValueType();

  // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
  SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
                                DAG.getConstant(0x8000, SL, SrcVT));
  return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
}
default:
  return SDValue();
}
4024}

4026SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
                                               DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
SDValue N0 = N->getOperand(0);

if (!N0.hasOneUse())
  return SDValue();

switch (N0.getOpcode()) {
case ISD::FP16_TO_FP: {
  assert(!Subtarget->has16BitInsts() && "should only see if f16 is illegal")(static_cast <bool> (!Subtarget->has16BitInsts() &&
 "should only see if f16 is illegal") ? void (0) : __assert_fail
 ("!Subtarget->has16BitInsts() && \"should only see if f16 is illegal\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 4036, __extension__
 __PRETTY_FUNCTION__));
  SDLoc SL(N);
  SDValue Src = N0.getOperand(0);
  EVT SrcVT = Src.getValueType();

  // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
  SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
                                DAG.getConstant(0x7fff, SL, SrcVT));
  return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
}
default:
  return SDValue();
}
4049}

4051SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
if (!CFP)
  return SDValue();

// XXX - Should this flush denormals?
const APFloat &Val = CFP->getValueAPF();
APFloat One(Val.getSemantics(), "1.0");
return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
4061}

4063SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
SDLoc DL(N);

switch(N->getOpcode()) {
1
Control jumps to 'case BFE_I32:'  at line 4170→
default:
  break;
case ISD::BITCAST: {
  EVT DestVT = N->getValueType(0);

  // Push casts through vector builds. This helps avoid emitting a large
  // number of copies when materializing floating point vector constants.
  //
  // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
  //   vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
  if (DestVT.isVector()) {
    SDValue Src = N->getOperand(0);
    if (Src.getOpcode() == ISD::BUILD_VECTOR) {
      EVT SrcVT = Src.getValueType();
      unsigned NElts = DestVT.getVectorNumElements();

      if (SrcVT.getVectorNumElements() == NElts) {
        EVT DestEltVT = DestVT.getVectorElementType();

        SmallVector<SDValue, 8> CastedElts;
        SDLoc SL(N);
        for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
          SDValue Elt = Src.getOperand(I);
          CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
        }

        return DAG.getBuildVector(DestVT, SL, CastedElts);
      }
    }
  }

  if (DestVT.getSizeInBits() != 64 || !DestVT.isVector())
    break;

  // Fold bitcasts of constants.
  //
  // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
  // TODO: Generalize and move to DAGCombiner
  SDValue Src = N->getOperand(0);
  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src)) {
    SDLoc SL(N);
    uint64_t CVal = C->getZExtValue();
    SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
                             DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
                             DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
    return DAG.getNode(ISD::BITCAST, SL, DestVT, BV);
  }

  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Src)) {
    const APInt &Val = C->getValueAPF().bitcastToAPInt();
    SDLoc SL(N);
    uint64_t CVal = Val.getZExtValue();
    SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
                              DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
                              DAG.getConstant(Hi_32(CVal), SL, MVT::i32));

    return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
  }

  break;
}
case ISD::SHL: {
  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
    break;

  return performShlCombine(N, DCI);
}
case ISD::SRL: {
  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
    break;

  return performSrlCombine(N, DCI);
}
case ISD::SRA: {
  if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
    break;

  return performSraCombine(N, DCI);
}
case ISD::TRUNCATE:
  return performTruncateCombine(N, DCI);
case ISD::MUL:
  return performMulCombine(N, DCI);
case ISD::SMUL_LOHI:
case ISD::UMUL_LOHI:
  return performMulLoHiCombine(N, DCI);
case ISD::MULHS:
  return performMulhsCombine(N, DCI);
case ISD::MULHU:
  return performMulhuCombine(N, DCI);
case AMDGPUISD::MUL_I24:
case AMDGPUISD::MUL_U24:
case AMDGPUISD::MULHI_I24:
case AMDGPUISD::MULHI_U24:
  return simplifyMul24(N, DCI);
case ISD::SELECT:
  return performSelectCombine(N, DCI);
case ISD::FNEG:
  return performFNegCombine(N, DCI);
case ISD::FABS:
  return performFAbsCombine(N, DCI);
case AMDGPUISD::BFE_I32:
case AMDGPUISD::BFE_U32: {
  assert(!N->getValueType(0).isVector() &&(static_cast <bool> (!N->getValueType(0).isVector() &&
 "Vector handling of BFE not implemented") ? void (0) : __assert_fail
 ("!N->getValueType(0).isVector() && \"Vector handling of BFE not implemented\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 4173, __extension__
 __PRETTY_FUNCTION__))
2
←
Assuming the condition is true→
3
←
'?' condition is true→
         "Vector handling of BFE not implemented")(static_cast <bool> (!N->getValueType(0).isVector() &&
 "Vector handling of BFE not implemented") ? void (0) : __assert_fail
 ("!N->getValueType(0).isVector() && \"Vector handling of BFE not implemented\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 4173, __extension__
 __PRETTY_FUNCTION__));
  ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
  if (!Width)
4
←
Assuming 'Width' is non-null→
5
←
Taking false branch→
    break;

  uint32_t WidthVal = Width->getZExtValue() & 0x1f;
  if (WidthVal == 0)
6
←
Assuming 'WidthVal' is not equal to 0→
7
←
Taking false branch→
    return DAG.getConstant(0, DL, MVT::i32);

  ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!Offset)
8
←
Assuming 'Offset' is non-null→
9
←
Taking false branch→
    break;

  SDValue BitsFrom = N->getOperand(0);
10
←
Value assigned to 'BitsFrom.Node'→
  uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;

  bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;

  if (OffsetVal == 0) {
11
←
Assuming 'OffsetVal' is not equal to 0→
12
←
Taking false branch→
    // This is already sign / zero extended, so try to fold away extra BFEs.
    unsigned SignBits =  Signed ? (32 - WidthVal + 1) : (32 - WidthVal);

    unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
    if (OpSignBits >= SignBits)
      return BitsFrom;

    EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
    if (Signed) {
      // This is a sign_extend_inreg. Replace it to take advantage of existing
      // DAG Combines. If not eliminated, we will match back to BFE during
      // selection.

      // TODO: The sext_inreg of extended types ends, although we can could
      // handle them in a single BFE.
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
                         DAG.getValueType(SmallVT));
    }

    return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
  }

  if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
13
←
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'→
28
←
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'→
29
←
Assuming 'CVal' is null→
    if (Signed) {
      return constantFoldBFE<int32_t>(DAG,
                                      CVal->getSExtValue(),
                                      OffsetVal,
                                      WidthVal,
                                      DL);
    }

    return constantFoldBFE<uint32_t>(DAG,
                                     CVal->getZExtValue(),
                                     OffsetVal,
                                     WidthVal,
                                     DL);
  }

  if ((OffsetVal + WidthVal) >= 32 &&
30
←
Assuming the condition is false→
      !(Subtarget->hasSDWA() && OffsetVal == 16 && WidthVal == 16)) {
    SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
    return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
                       BitsFrom, ShiftVal);
  }

  if (BitsFrom.hasOneUse()) {
31
←
Calling 'SDValue::hasOneUse'→
    APInt Demanded = APInt::getBitsSet(32,
                                       OffsetVal,
                                       OffsetVal + WidthVal);

    KnownBits Known;
    TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                                          !DCI.isBeforeLegalizeOps());
    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    if (TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) ||
        TLI.SimplifyDemandedBits(BitsFrom, Demanded, Known, TLO)) {
      DCI.CommitTargetLoweringOpt(TLO);
    }
  }

  break;
}
case ISD::LOAD:
  return performLoadCombine(N, DCI);
case ISD::STORE:
  return performStoreCombine(N, DCI);
case AMDGPUISD::RCP:
case AMDGPUISD::RCP_IFLAG:
  return performRcpCombine(N, DCI);
case ISD::AssertZext:
case ISD::AssertSext:
  return performAssertSZExtCombine(N, DCI);
case ISD::INTRINSIC_WO_CHAIN:
  return performIntrinsicWOChainCombine(N, DCI);
}
return SDValue();
4268}

4270//===----------------------------------------------------------------------===//
4271// Helper functions
4272//===----------------------------------------------------------------------===//

4274SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
                                                 const TargetRegisterClass *RC,
                                                 Register Reg, EVT VT,
                                                 const SDLoc &SL,
                                                 bool RawReg) const {
MachineFunction &MF = DAG.getMachineFunction();
MachineRegisterInfo &MRI = MF.getRegInfo();
Register VReg;

if (!MRI.isLiveIn(Reg)) {
  VReg = MRI.createVirtualRegister(RC);
  MRI.addLiveIn(Reg, VReg);
} else {
  VReg = MRI.getLiveInVirtReg(Reg);
}

if (RawReg)
  return DAG.getRegister(VReg, VT);

return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
4294}

4296// This may be called multiple times, and nothing prevents creating multiple
4297// objects at the same offset. See if we already defined this object.
4298static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
                                     int64_t Offset) {
for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
  if (MFI.getObjectOffset(I) == Offset) {
    assert(MFI.getObjectSize(I) == Size)(static_cast <bool> (MFI.getObjectSize(I) == Size) ? void
 (0) : __assert_fail ("MFI.getObjectSize(I) == Size", "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp"
, 4302, __extension__ __PRETTY_FUNCTION__));
    return I;
  }
}

return MFI.CreateFixedObject(Size, Offset, true);
4308}

4310SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
                                                EVT VT,
                                                const SDLoc &SL,
                                                int64_t Offset) const {
MachineFunction &MF = DAG.getMachineFunction();
MachineFrameInfo &MFI = MF.getFrameInfo();
int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset);

auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);

return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, Align(4),
                   MachineMemOperand::MODereferenceable |
                       MachineMemOperand::MOInvariant);
4324}

4326SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
                                                 const SDLoc &SL,
                                                 SDValue Chain,
                                                 SDValue ArgVal,
                                                 int64_t Offset) const {
MachineFunction &MF = DAG.getMachineFunction();
MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();

SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
// Stores to the argument stack area are relative to the stack pointer.
SDValue SP =
    DAG.getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32);
Ptr = DAG.getNode(ISD::ADD, SL, MVT::i32, SP, Ptr);
SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, Align(4),
                             MachineMemOperand::MODereferenceable);
return Store;
4343}

4345SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
                                           const TargetRegisterClass *RC,
                                           EVT VT, const SDLoc &SL,
                                           const ArgDescriptor &Arg) const {
assert(Arg && "Attempting to load missing argument")(static_cast <bool> (Arg && "Attempting to load missing argument"
) ? void (0) : __assert_fail ("Arg && \"Attempting to load missing argument\""
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 4349, __extension__
 __PRETTY_FUNCTION__));

SDValue V = Arg.isRegister() ?
  CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
  loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());

if (!Arg.isMasked())
  return V;

unsigned Mask = Arg.getMask();
unsigned Shift = countTrailingZeros<unsigned>(Mask);
V = DAG.getNode(ISD::SRL, SL, VT, V,
                DAG.getShiftAmountConstant(Shift, VT, SL));
return DAG.getNode(ISD::AND, SL, VT, V,
                   DAG.getConstant(Mask >> Shift, SL, VT));
4364}

4366uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
  const MachineFunction &MF, const ImplicitParameter Param) const {
const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
const AMDGPUSubtarget &ST =
    AMDGPUSubtarget::get(getTargetMachine(), MF.getFunction());
unsigned ExplicitArgOffset = ST.getExplicitKernelArgOffset(MF.getFunction());
const Align Alignment = ST.getAlignmentForImplicitArgPtr();
uint64_t ArgOffset = alignTo(MFI->getExplicitKernArgSize(), Alignment) +
                     ExplicitArgOffset;
switch (Param) {
case FIRST_IMPLICIT:
  return ArgOffset;
case PRIVATE_BASE:
  return ArgOffset + AMDGPU::ImplicitArg::PRIVATE_BASE_OFFSET;
case SHARED_BASE:
  return ArgOffset + AMDGPU::ImplicitArg::SHARED_BASE_OFFSET;
case QUEUE_PTR:
  return ArgOffset + AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET;
}
llvm_unreachable("unexpected implicit parameter type")::llvm::llvm_unreachable_internal("unexpected implicit parameter type"
, "llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp", 4385);
4386}

4388#define NODE_NAME_CASE(node)case AMDGPUISD::node: return "node"; case AMDGPUISD::node: return #node;

4390const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
switch ((AMDGPUISD::NodeType)Opcode) {
case AMDGPUISD::FIRST_NUMBER: break;
// AMDIL DAG nodes
NODE_NAME_CASE(UMUL)case AMDGPUISD::UMUL: return "UMUL";;
NODE_NAME_CASE(BRANCH_COND)case AMDGPUISD::BRANCH_COND: return "BRANCH_COND";;

// AMDGPU DAG nodes
NODE_NAME_CASE(IF)case AMDGPUISD::IF: return "IF";
NODE_NAME_CASE(ELSE)case AMDGPUISD::ELSE: return "ELSE";
NODE_NAME_CASE(LOOP)case AMDGPUISD::LOOP: return "LOOP";
NODE_NAME_CASE(CALL)case AMDGPUISD::CALL: return "CALL";
NODE_NAME_CASE(TC_RETURN)case AMDGPUISD::TC_RETURN: return "TC_RETURN";
NODE_NAME_CASE(TRAP)case AMDGPUISD::TRAP: return "TRAP";
NODE_NAME_CASE(RET_FLAG)case AMDGPUISD::RET_FLAG: return "RET_FLAG";
NODE_NAME_CASE(RETURN_TO_EPILOG)case AMDGPUISD::RETURN_TO_EPILOG: return "RETURN_TO_EPILOG";
NODE_NAME_CASE(ENDPGM)case AMDGPUISD::ENDPGM: return "ENDPGM";
NODE_NAME_CASE(DWORDADDR)case AMDGPUISD::DWORDADDR: return "DWORDADDR";
NODE_NAME_CASE(FRACT)case AMDGPUISD::FRACT: return "FRACT";
NODE_NAME_CASE(SETCC)case AMDGPUISD::SETCC: return "SETCC";
NODE_NAME_CASE(SETREG)case AMDGPUISD::SETREG: return "SETREG";
NODE_NAME_CASE(DENORM_MODE)case AMDGPUISD::DENORM_MODE: return "DENORM_MODE";
NODE_NAME_CASE(FMA_W_CHAIN)case AMDGPUISD::FMA_W_CHAIN: return "FMA_W_CHAIN";
NODE_NAME_CASE(FMUL_W_CHAIN)case AMDGPUISD::FMUL_W_CHAIN: return "FMUL_W_CHAIN";
NODE_NAME_CASE(CLAMP)case AMDGPUISD::CLAMP: return "CLAMP";
NODE_NAME_CASE(COS_HW)case AMDGPUISD::COS_HW: return "COS_HW";
NODE_NAME_CASE(SIN_HW)case AMDGPUISD::SIN_HW: return "SIN_HW";
NODE_NAME_CASE(FMAX_LEGACY)case AMDGPUISD::FMAX_LEGACY: return "FMAX_LEGACY";
NODE_NAME_CASE(FMIN_LEGACY)case AMDGPUISD::FMIN_LEGACY: return "FMIN_LEGACY";
NODE_NAME_CASE(FMAX3)case AMDGPUISD::FMAX3: return "FMAX3";
NODE_NAME_CASE(SMAX3)case AMDGPUISD::SMAX3: return "SMAX3";
NODE_NAME_CASE(UMAX3)case AMDGPUISD::UMAX3: return "UMAX3";
NODE_NAME_CASE(FMIN3)case AMDGPUISD::FMIN3: return "FMIN3";
NODE_NAME_CASE(SMIN3)case AMDGPUISD::SMIN3: return "SMIN3";
NODE_NAME_CASE(UMIN3)case AMDGPUISD::UMIN3: return "UMIN3";
NODE_NAME_CASE(FMED3)case AMDGPUISD::FMED3: return "FMED3";
NODE_NAME_CASE(SMED3)case AMDGPUISD::SMED3: return "SMED3";
NODE_NAME_CASE(UMED3)case AMDGPUISD::UMED3: return "UMED3";
NODE_NAME_CASE(FDOT2)case AMDGPUISD::FDOT2: return "FDOT2";
NODE_NAME_CASE(URECIP)case AMDGPUISD::URECIP: return "URECIP";
NODE_NAME_CASE(DIV_SCALE)case AMDGPUISD::DIV_SCALE: return "DIV_SCALE";
NODE_NAME_CASE(DIV_FMAS)case AMDGPUISD::DIV_FMAS: return "DIV_FMAS";
NODE_NAME_CASE(DIV_FIXUP)case AMDGPUISD::DIV_FIXUP: return "DIV_FIXUP";
NODE_NAME_CASE(FMAD_FTZ)case AMDGPUISD::FMAD_FTZ: return "FMAD_FTZ";
NODE_NAME_CASE(RCP)case AMDGPUISD::RCP: return "RCP";
NODE_NAME_CASE(RSQ)case AMDGPUISD::RSQ: return "RSQ";
NODE_NAME_CASE(RCP_LEGACY)case AMDGPUISD::RCP_LEGACY: return "RCP_LEGACY";
NODE_NAME_CASE(RCP_IFLAG)case AMDGPUISD::RCP_IFLAG: return "RCP_IFLAG";
NODE_NAME_CASE(FMUL_LEGACY)case AMDGPUISD::FMUL_LEGACY: return "FMUL_LEGACY";
NODE_NAME_CASE(RSQ_CLAMP)case AMDGPUISD::RSQ_CLAMP: return "RSQ_CLAMP";
NODE_NAME_CASE(LDEXP)case AMDGPUISD::LDEXP: return "LDEXP";
NODE_NAME_CASE(FP_CLASS)case AMDGPUISD::FP_CLASS: return "FP_CLASS";
NODE_NAME_CASE(DOT4)case AMDGPUISD::DOT4: return "DOT4";
NODE_NAME_CASE(CARRY)case AMDGPUISD::CARRY: return "CARRY";
NODE_NAME_CASE(BORROW)case AMDGPUISD::BORROW: return "BORROW";
NODE_NAME_CASE(BFE_U32)case AMDGPUISD::BFE_U32: return "BFE_U32";
NODE_NAME_CASE(BFE_I32)case AMDGPUISD::BFE_I32: return "BFE_I32";
NODE_NAME_CASE(BFI)case AMDGPUISD::BFI: return "BFI";
NODE_NAME_CASE(BFM)case AMDGPUISD::BFM: return "BFM";
NODE_NAME_CASE(FFBH_U32)case AMDGPUISD::FFBH_U32: return "FFBH_U32";
NODE_NAME_CASE(FFBH_I32)case AMDGPUISD::FFBH_I32: return "FFBH_I32";
NODE_NAME_CASE(FFBL_B32)case AMDGPUISD::FFBL_B32: return "FFBL_B32";
NODE_NAME_CASE(MUL_U24)case AMDGPUISD::MUL_U24: return "MUL_U24";
NODE_NAME_CASE(MUL_I24)case AMDGPUISD::MUL_I24: return "MUL_I24";
NODE_NAME_CASE(MULHI_U24)case AMDGPUISD::MULHI_U24: return "MULHI_U24";
NODE_NAME_CASE(MULHI_I24)case AMDGPUISD::MULHI_I24: return "MULHI_I24";
NODE_NAME_CASE(MAD_U24)case AMDGPUISD::MAD_U24: return "MAD_U24";
NODE_NAME_CASE(MAD_I24)case AMDGPUISD::MAD_I24: return "MAD_I24";
NODE_NAME_CASE(MAD_I64_I32)case AMDGPUISD::MAD_I64_I32: return "MAD_I64_I32";
NODE_NAME_CASE(MAD_U64_U32)case AMDGPUISD::MAD_U64_U32: return "MAD_U64_U32";
NODE_NAME_CASE(PERM)case AMDGPUISD::PERM: return "PERM";
NODE_NAME_CASE(TEXTURE_FETCH)case AMDGPUISD::TEXTURE_FETCH: return "TEXTURE_FETCH";
NODE_NAME_CASE(R600_EXPORT)case AMDGPUISD::R600_EXPORT: return "R600_EXPORT";
NODE_NAME_CASE(CONST_ADDRESS)case AMDGPUISD::CONST_ADDRESS: return "CONST_ADDRESS";
NODE_NAME_CASE(REGISTER_LOAD)case AMDGPUISD::REGISTER_LOAD: return "REGISTER_LOAD";
NODE_NAME_CASE(REGISTER_STORE)case AMDGPUISD::REGISTER_STORE: return "REGISTER_STORE";
NODE_NAME_CASE(SAMPLE)case AMDGPUISD::SAMPLE: return "SAMPLE";
NODE_NAME_CASE(SAMPLEB)case AMDGPUISD::SAMPLEB: return "SAMPLEB";
NODE_NAME_CASE(SAMPLED)case AMDGPUISD::SAMPLED: return "SAMPLED";
NODE_NAME_CASE(SAMPLEL)case AMDGPUISD::SAMPLEL: return "SAMPLEL";
NODE_NAME_CASE(CVT_F32_UBYTE0)case AMDGPUISD::CVT_F32_UBYTE0: return "CVT_F32_UBYTE0";
NODE_NAME_CASE(CVT_F32_UBYTE1)case AMDGPUISD::CVT_F32_UBYTE1: return "CVT_F32_UBYTE1";
NODE_NAME_CASE(CVT_F32_UBYTE2)case AMDGPUISD::CVT_F32_UBYTE2: return "CVT_F32_UBYTE2";
NODE_NAME_CASE(CVT_F32_UBYTE3)case AMDGPUISD::CVT_F32_UBYTE3: return "CVT_F32_UBYTE3";
NODE_NAME_CASE(CVT_PKRTZ_F16_F32)case AMDGPUISD::CVT_PKRTZ_F16_F32: return "CVT_PKRTZ_F16_F32"
;
NODE_NAME_CASE(CVT_PKNORM_I16_F32)case AMDGPUISD::CVT_PKNORM_I16_F32: return "CVT_PKNORM_I16_F32"
;
NODE_NAME_CASE(CVT_PKNORM_U16_F32)case AMDGPUISD::CVT_PKNORM_U16_F32: return "CVT_PKNORM_U16_F32"
;
NODE_NAME_CASE(CVT_PK_I16_I32)case AMDGPUISD::CVT_PK_I16_I32: return "CVT_PK_I16_I32";
NODE_NAME_CASE(CVT_PK_U16_U32)case AMDGPUISD::CVT_PK_U16_U32: return "CVT_PK_U16_U32";
NODE_NAME_CASE(FP_TO_FP16)case AMDGPUISD::FP_TO_FP16: return "FP_TO_FP16";
NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)case AMDGPUISD::BUILD_VERTICAL_VECTOR: return "BUILD_VERTICAL_VECTOR"
;
NODE_NAME_CASE(CONST_DATA_PTR)case AMDGPUISD::CONST_DATA_PTR: return "CONST_DATA_PTR";
NODE_NAME_CASE(PC_ADD_REL_OFFSET)case AMDGPUISD::PC_ADD_REL_OFFSET: return "PC_ADD_REL_OFFSET"
;
NODE_NAME_CASE(LDS)case AMDGPUISD::LDS: return "LDS";
NODE_NAME_CASE(FPTRUNC_ROUND_UPWARD)case AMDGPUISD::FPTRUNC_ROUND_UPWARD: return "FPTRUNC_ROUND_UPWARD"
;
NODE_NAME_CASE(FPTRUNC_ROUND_DOWNWARD)case AMDGPUISD::FPTRUNC_ROUND_DOWNWARD: return "FPTRUNC_ROUND_DOWNWARD"
;
NODE_NAME_CASE(DUMMY_CHAIN)case AMDGPUISD::DUMMY_CHAIN: return "DUMMY_CHAIN";
case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
NODE_NAME_CASE(LOAD_D16_HI)case AMDGPUISD::LOAD_D16_HI: return "LOAD_D16_HI";
NODE_NAME_CASE(LOAD_D16_LO)case AMDGPUISD::LOAD_D16_LO: return "LOAD_D16_LO";
NODE_NAME_CASE(LOAD_D16_HI_I8)case AMDGPUISD::LOAD_D16_HI_I8: return "LOAD_D16_HI_I8";
NODE_NAME_CASE(LOAD_D16_HI_U8)case AMDGPUISD::LOAD_D16_HI_U8: return "LOAD_D16_HI_U8";
NODE_NAME_CASE(LOAD_D16_LO_I8)case AMDGPUISD::LOAD_D16_LO_I8: return "LOAD_D16_LO_I8";
NODE_NAME_CASE(LOAD_D16_LO_U8)case AMDGPUISD::LOAD_D16_LO_U8: return "LOAD_D16_LO_U8";
NODE_NAME_CASE(STORE_MSKOR)case AMDGPUISD::STORE_MSKOR: return "STORE_MSKOR";
NODE_NAME_CASE(LOAD_CONSTANT)case AMDGPUISD::LOAD_CONSTANT: return "LOAD_CONSTANT";
NODE_NAME_CASE(TBUFFER_STORE_FORMAT)case AMDGPUISD::TBUFFER_STORE_FORMAT: return "TBUFFER_STORE_FORMAT"
;
NODE_NAME_CASE(TBUFFER_STORE_FORMAT_D16)case AMDGPUISD::TBUFFER_STORE_FORMAT_D16: return "TBUFFER_STORE_FORMAT_D16"
;
NODE_NAME_CASE(TBUFFER_LOAD_FORMAT)case AMDGPUISD::TBUFFER_LOAD_FORMAT: return "TBUFFER_LOAD_FORMAT"
;
NODE_NAME_CASE(TBUFFER_LOAD_FORMAT_D16)case AMDGPUISD::TBUFFER_LOAD_FORMAT_D16: return "TBUFFER_LOAD_FORMAT_D16"
;
NODE_NAME_CASE(DS_ORDERED_COUNT)case AMDGPUISD::DS_ORDERED_COUNT: return "DS_ORDERED_COUNT";
NODE_NAME_CASE(ATOMIC_CMP_SWAP)case AMDGPUISD::ATOMIC_CMP_SWAP: return "ATOMIC_CMP_SWAP";
NODE_NAME_CASE(ATOMIC_INC)case AMDGPUISD::ATOMIC_INC: return "ATOMIC_INC";
NODE_NAME_CASE(ATOMIC_DEC)case AMDGPUISD::ATOMIC_DEC: return "ATOMIC_DEC";
NODE_NAME_CASE(ATOMIC_LOAD_FMIN)case AMDGPUISD::ATOMIC_LOAD_FMIN: return "ATOMIC_LOAD_FMIN";
NODE_NAME_CASE(ATOMIC_LOAD_FMAX)case AMDGPUISD::ATOMIC_LOAD_FMAX: return "ATOMIC_LOAD_FMAX";
NODE_NAME_CASE(BUFFER_LOAD)case AMDGPUISD::BUFFER_LOAD: return "BUFFER_LOAD";
NODE_NAME_CASE(BUFFER_LOAD_UBYTE)case AMDGPUISD::BUFFER_LOAD_UBYTE: return "BUFFER_LOAD_UBYTE"
;
NODE_NAME_CASE(BUFFER_LOAD_USHORT)case AMDGPUISD::BUFFER_LOAD_USHORT: return "BUFFER_LOAD_USHORT"
;
NODE_NAME_CASE(BUFFER_LOAD_BYTE)case AMDGPUISD::BUFFER_LOAD_BYTE: return "BUFFER_LOAD_BYTE";
NODE_NAME_CASE(BUFFER_LOAD_SHORT)case AMDGPUISD::BUFFER_LOAD_SHORT: return "BUFFER_LOAD_SHORT"
;
NODE_NAME_CASE(BUFFER_LOAD_FORMAT)case AMDGPUISD::BUFFER_LOAD_FORMAT: return "BUFFER_LOAD_FORMAT"
;
NODE_NAME_CASE(BUFFER_LOAD_FORMAT_D16)case AMDGPUISD::BUFFER_LOAD_FORMAT_D16: return "BUFFER_LOAD_FORMAT_D16"
;
NODE_NAME_CASE(SBUFFER_LOAD)case AMDGPUISD::SBUFFER_LOAD: return "SBUFFER_LOAD";
NODE_NAME_CASE(BUFFER_STORE)case AMDGPUISD::BUFFER_STORE: return "BUFFER_STORE";
NODE_NAME_CASE(BUFFER_STORE_BYTE)case AMDGPUISD::BUFFER_STORE_BYTE: return "BUFFER_STORE_BYTE"
;
NODE_NAME_CASE(BUFFER_STORE_SHORT)case AMDGPUISD::BUFFER_STORE_SHORT: return "BUFFER_STORE_SHORT"
;
NODE_NAME_CASE(BUFFER_STORE_FORMAT)case AMDGPUISD::BUFFER_STORE_FORMAT: return "BUFFER_STORE_FORMAT"
;
NODE_NAME_CASE(BUFFER_STORE_FORMAT_D16)case AMDGPUISD::BUFFER_STORE_FORMAT_D16: return "BUFFER_STORE_FORMAT_D16"
;
NODE_NAME_CASE(BUFFER_ATOMIC_SWAP)case AMDGPUISD::BUFFER_ATOMIC_SWAP: return "BUFFER_ATOMIC_SWAP"
;
NODE_NAME_CASE(BUFFER_ATOMIC_ADD)case AMDGPUISD::BUFFER_ATOMIC_ADD: return "BUFFER_ATOMIC_ADD"
;
NODE_NAME_CASE(BUFFER_ATOMIC_SUB)case AMDGPUISD::BUFFER_ATOMIC_SUB: return "BUFFER_ATOMIC_SUB"
;
NODE_NAME_CASE(BUFFER_ATOMIC_SMIN)case AMDGPUISD::BUFFER_ATOMIC_SMIN: return "BUFFER_ATOMIC_SMIN"
;
NODE_NAME_CASE(BUFFER_ATOMIC_UMIN)case AMDGPUISD::BUFFER_ATOMIC_UMIN: return "BUFFER_ATOMIC_UMIN"
;
NODE_NAME_CASE(BUFFER_ATOMIC_SMAX)case AMDGPUISD::BUFFER_ATOMIC_SMAX: return "BUFFER_ATOMIC_SMAX"
;
NODE_NAME_CASE(BUFFER_ATOMIC_UMAX)case AMDGPUISD::BUFFER_ATOMIC_UMAX: return "BUFFER_ATOMIC_UMAX"
;
NODE_NAME_CASE(BUFFER_ATOMIC_AND)case AMDGPUISD::BUFFER_ATOMIC_AND: return "BUFFER_ATOMIC_AND"
;
NODE_NAME_CASE(BUFFER_ATOMIC_OR)case AMDGPUISD::BUFFER_ATOMIC_OR: return "BUFFER_ATOMIC_OR";
NODE_NAME_CASE(BUFFER_ATOMIC_XOR)case AMDGPUISD::BUFFER_ATOMIC_XOR: return "BUFFER_ATOMIC_XOR"
;
NODE_NAME_CASE(BUFFER_ATOMIC_INC)case AMDGPUISD::BUFFER_ATOMIC_INC: return "BUFFER_ATOMIC_INC"
;
NODE_NAME_CASE(BUFFER_ATOMIC_DEC)case AMDGPUISD::BUFFER_ATOMIC_DEC: return "BUFFER_ATOMIC_DEC"
;
NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP)case AMDGPUISD::BUFFER_ATOMIC_CMPSWAP: return "BUFFER_ATOMIC_CMPSWAP"
;
NODE_NAME_CASE(BUFFER_ATOMIC_CSUB)case AMDGPUISD::BUFFER_ATOMIC_CSUB: return "BUFFER_ATOMIC_CSUB"
;
NODE_NAME_CASE(BUFFER_ATOMIC_FADD)case AMDGPUISD::BUFFER_ATOMIC_FADD: return "BUFFER_ATOMIC_FADD"
;
NODE_NAME_CASE(BUFFER_ATOMIC_FMIN)case AMDGPUISD::BUFFER_ATOMIC_FMIN: return "BUFFER_ATOMIC_FMIN"
;
NODE_NAME_CASE(BUFFER_ATOMIC_FMAX)case AMDGPUISD::BUFFER_ATOMIC_FMAX: return "BUFFER_ATOMIC_FMAX"
;

case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
}
return nullptr;
4540}

4542SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
                                            SelectionDAG &DAG, int Enabled,
                                            int &RefinementSteps,
                                            bool &UseOneConstNR,
                                            bool Reciprocal) const {
EVT VT = Operand.getValueType();

if (VT == MVT::f32) {
  RefinementSteps = 0;
  return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
}

// TODO: There is also f64 rsq instruction, but the documentation is less
// clear on its precision.

return SDValue();
4558}

4560SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
                                             SelectionDAG &DAG, int Enabled,
                                             int &RefinementSteps) const {
EVT VT = Operand.getValueType();

if (VT == MVT::f32) {
  // Reciprocal, < 1 ulp error.
  //
  // This reciprocal approximation converges to < 0.5 ulp error with one
  // newton rhapson performed with two fused multiple adds (FMAs).

  RefinementSteps = 0;
  return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
}

// TODO: There is also f64 rcp instruction, but the documentation is less
// clear on its precision.

return SDValue();
4579}

4581void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
  const SDValue Op, KnownBits &Known,
  const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {

Known.resetAll(); // Don't know anything.

unsigned Opc = Op.getOpcode();

switch (Opc) {
default:
  break;
case AMDGPUISD::CARRY:
case AMDGPUISD::BORROW: {
  Known.Zero = APInt::getHighBitsSet(32, 31);
  break;
}

case AMDGPUISD::BFE_I32:
case AMDGPUISD::BFE_U32: {
  ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  if (!CWidth)
    return;

  uint32_t Width = CWidth->getZExtValue() & 0x1f;

  if (Opc == AMDGPUISD::BFE_U32)
    Known.Zero = APInt::getHighBitsSet(32, 32 - Width);

  break;
}
case AMDGPUISD::FP_TO_FP16: {
  unsigned BitWidth = Known.getBitWidth();

  // High bits are zero.
  Known.Zero = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
  break;
}
case AMDGPUISD::MUL_U24:
case AMDGPUISD::MUL_I24: {
  KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
  KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
  unsigned TrailZ = LHSKnown.countMinTrailingZeros() +
                    RHSKnown.countMinTrailingZeros();
  Known.Zero.setLowBits(std::min(TrailZ, 32u));
  // Skip extra check if all bits are known zeros.
  if (TrailZ >= 32)
    break;

  // Truncate to 24 bits.
  LHSKnown = LHSKnown.trunc(24);
  RHSKnown = RHSKnown.trunc(24);

  if (Opc == AMDGPUISD::MUL_I24) {
    unsigned LHSValBits = LHSKnown.countMaxSignificantBits();
    unsigned RHSValBits = RHSKnown.countMaxSignificantBits();
    unsigned MaxValBits = LHSValBits + RHSValBits;
    if (MaxValBits > 32)
      break;
    unsigned SignBits = 32 - MaxValBits + 1;
    bool LHSNegative = LHSKnown.isNegative();
    bool LHSNonNegative = LHSKnown.isNonNegative();
    bool LHSPositive = LHSKnown.isStrictlyPositive();
    bool RHSNegative = RHSKnown.isNegative();
    bool RHSNonNegative = RHSKnown.isNonNegative();
    bool RHSPositive = RHSKnown.isStrictlyPositive();

    if ((LHSNonNegative && RHSNonNegative) || (LHSNegative && RHSNegative))
      Known.Zero.setHighBits(SignBits);
    else if ((LHSNegative && RHSPositive) || (LHSPositive && RHSNegative))
      Known.One.setHighBits(SignBits);
  } else {
    unsigned LHSValBits = LHSKnown.countMaxActiveBits();
    unsigned RHSValBits = RHSKnown.countMaxActiveBits();
    unsigned MaxValBits = LHSValBits + RHSValBits;
    if (MaxValBits >= 32)
      break;
    Known.Zero.setBitsFrom(MaxValBits);
  }
  break;
}
case AMDGPUISD::PERM: {
  ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  if (!CMask)
    return;

  KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
  KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
  unsigned Sel = CMask->getZExtValue();

  for (unsigned I = 0; I < 32; I += 8) {
    unsigned SelBits = Sel & 0xff;
    if (SelBits < 4) {
      SelBits *= 8;
      Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
      Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
    } else if (SelBits < 7) {
      SelBits = (SelBits & 3) * 8;
      Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
      Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
    } else if (SelBits == 0x0c) {
      Known.Zero |= 0xFFull << I;
    } else if (SelBits > 0x0c) {
      Known.One |= 0xFFull << I;
    }
    Sel >>= 8;
  }
  break;
}
case AMDGPUISD::BUFFER_LOAD_UBYTE:  {
  Known.Zero.setHighBits(24);
  break;
}
case AMDGPUISD::BUFFER_LOAD_USHORT: {
  Known.Zero.setHighBits(16);
  break;
}
case AMDGPUISD::LDS: {
  auto GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
  Align Alignment = GA->getGlobal()->getPointerAlignment(DAG.getDataLayout());

  Known.Zero.setHighBits(16);
  Known.Zero.setLowBits(Log2(Alignment));
  break;
}
case ISD::INTRINSIC_WO_CHAIN: {
  unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
  switch (IID) {
  case Intrinsic::amdgcn_mbcnt_lo:
  case Intrinsic::amdgcn_mbcnt_hi: {
    const GCNSubtarget &ST =
        DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
    // These return at most the wavefront size - 1.
    unsigned Size = Op.getValueType().getSizeInBits();
    Known.Zero.setHighBits(Size - ST.getWavefrontSizeLog2());
    break;
  }
  default:
    break;
  }
}
}
4722}

4724unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
  unsigned Depth) const {
switch (Op.getOpcode()) {
case AMDGPUISD::BFE_I32: {
  ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  if (!Width)
    return 1;

  unsigned SignBits = 32 - Width->getZExtValue() + 1;
  if (!isNullConstant(Op.getOperand(1)))
    return SignBits;

  // TODO: Could probably figure something out with non-0 offsets.
  unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
  return std::max(SignBits, Op0SignBits);
}

case AMDGPUISD::BFE_U32: {
  ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
}

case AMDGPUISD::CARRY:
case AMDGPUISD::BORROW:
  return 31;
case AMDGPUISD::BUFFER_LOAD_BYTE:
  return 25;
case AMDGPUISD::BUFFER_LOAD_SHORT:
  return 17;
case AMDGPUISD::BUFFER_LOAD_UBYTE:
  return 24;
case AMDGPUISD::BUFFER_LOAD_USHORT:
  return 16;
case AMDGPUISD::FP_TO_FP16:
  return 16;
default:
  return 1;
}
4763}

4765unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
GISelKnownBits &Analysis, Register R,
const APInt &DemandedElts, const MachineRegisterInfo &MRI,
unsigned Depth) const {
const MachineInstr *MI = MRI.getVRegDef(R);
if (!MI)
  return 1;

// TODO: Check range metadata on MMO.
switch (MI->getOpcode()) {
case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
  return 25;
case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
  return 17;
case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
  return 24;
case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
  return 16;
default:
  return 1;
}
4786}

4788bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
                                                      const SelectionDAG &DAG,
                                                      bool SNaN,
                                                      unsigned Depth) const {
unsigned Opcode = Op.getOpcode();
switch (Opcode) {
case AMDGPUISD::FMIN_LEGACY:
case AMDGPUISD::FMAX_LEGACY: {
  if (SNaN)
    return true;

  // TODO: Can check no nans on one of the operands for each one, but which
  // one?
  return false;
}
case AMDGPUISD::FMUL_LEGACY:
case AMDGPUISD::CVT_PKRTZ_F16_F32: {
  if (SNaN)
    return true;
  return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
         DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
}
case AMDGPUISD::FMED3:
case AMDGPUISD::FMIN3:
case AMDGPUISD::FMAX3:
case AMDGPUISD::FMAD_FTZ: {
  if (SNaN)
    return true;
  return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
         DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
         DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
}
case AMDGPUISD::CVT_F32_UBYTE0:
case AMDGPUISD::CVT_F32_UBYTE1:
case AMDGPUISD::CVT_F32_UBYTE2:
case AMDGPUISD::CVT_F32_UBYTE3:
  return true;

case AMDGPUISD::RCP:
case AMDGPUISD::RSQ:
case AMDGPUISD::RCP_LEGACY:
case AMDGPUISD::RSQ_CLAMP: {
  if (SNaN)
    return true;

  // TODO: Need is known positive check.
  return false;
}
case AMDGPUISD::LDEXP:
case AMDGPUISD::FRACT: {
  if (SNaN)
    return true;
  return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
}
case AMDGPUISD::DIV_SCALE:
case AMDGPUISD::DIV_FMAS:
case AMDGPUISD::DIV_FIXUP:
  // TODO: Refine on operands.
  return SNaN;
case AMDGPUISD::SIN_HW:
case AMDGPUISD::COS_HW: {
  // TODO: Need check for infinity
  return SNaN;
}
case ISD::INTRINSIC_WO_CHAIN: {
  unsigned IntrinsicID
    = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
  // TODO: Handle more intrinsics
  switch (IntrinsicID) {
  case Intrinsic::amdgcn_cubeid:
    return true;

  case Intrinsic::amdgcn_frexp_mant: {
    if (SNaN)
      return true;
    return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
  }
  case Intrinsic::amdgcn_cvt_pkrtz: {
    if (SNaN)
      return true;
    return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
           DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
  }
  case Intrinsic::amdgcn_rcp:
  case Intrinsic::amdgcn_rsq:
  case Intrinsic::amdgcn_rcp_legacy:
  case Intrinsic::amdgcn_rsq_legacy:
  case Intrinsic::amdgcn_rsq_clamp: {
    if (SNaN)
      return true;

    // TODO: Need is known positive check.
    return false;
  }
  case Intrinsic::amdgcn_trig_preop:
  case Intrinsic::amdgcn_fdot2:
    // TODO: Refine on operand
    return SNaN;
  case Intrinsic::amdgcn_fma_legacy:
    if (SNaN)
      return true;
    return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
           DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1) &&
           DAG.isKnownNeverNaN(Op.getOperand(3), SNaN, Depth + 1);
  default:
    return false;
  }
}
default:
  return false;
}
4899}

4901TargetLowering::AtomicExpansionKind
4902AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
switch (RMW->getOperation()) {
case AtomicRMWInst::Nand:
case AtomicRMWInst::FAdd:
case AtomicRMWInst::FSub:
  return AtomicExpansionKind::CmpXChg;
default:
  return AtomicExpansionKind::None;
}
4911}

4913bool AMDGPUTargetLowering::isConstantUnsignedBitfieldExtractLegal(
  unsigned Opc, LLT Ty1, LLT Ty2) const {
return (Ty1 == LLT::scalar(32) || Ty1 == LLT::scalar(64)) &&
       Ty2 == LLT::scalar(32);
4917}

←

/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/Support/Casting.h

→

1//===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the isa<X>(), cast<X>(), dyn_cast<X>(), cast_or_null<X>(),
10// and dyn_cast_or_null<X>() templates.
11//
12//===----------------------------------------------------------------------===//
13 
14#ifndef LLVM_SUPPORT_CASTING_H
15#define LLVM_SUPPORT_CASTING_H
16 
17#include "llvm/Support/Compiler.h"
18#include "llvm/Support/type_traits.h"
19#include <cassert>
20#include <memory>
21#include <type_traits>
22 
23namespace llvm {
24 
25//===----------------------------------------------------------------------===//
26//                          isa<x> Support Templates
27//===----------------------------------------------------------------------===//
28 
29// Define a template that can be specialized by smart pointers to reflect the
30// fact that they are automatically dereferenced, and are not involved with the
31// template selection process...  the default implementation is a noop.
32//
33template<typename From> struct simplify_type {
34  using SimpleType = From; // The real type this represents...
35 
36  // An accessor to get the real value...
37  static SimpleType &getSimplifiedValue(From &Val) { return Val; }
38};
39 
40template<typename From> struct simplify_type<const From> {
41  using NonConstSimpleType = typename simplify_type<From>::SimpleType;
42  using SimpleType =
43      typename add_const_past_pointer<NonConstSimpleType>::type;
44  using RetType =
45      typename add_lvalue_reference_if_not_pointer<SimpleType>::type;
46 
47  static RetType getSimplifiedValue(const From& Val) {
48    return simplify_type<From>::getSimplifiedValue(const_cast<From&>(Val));
49  }
50};
51 
52// The core of the implementation of isa<X> is here; To and From should be
53// the names of classes.  This template can be specialized to customize the
54// implementation of isa<> without rewriting it from scratch.
55template <typename To, typename From, typename Enabler = void>
56struct isa_impl {
57  static inline bool doit(const From &Val) {
58    return To::classof(&Val);
59  }
60};
61 
62/// Always allow upcasts, and perform no dynamic check for them.
63template <typename To, typename From>
64struct isa_impl<To, From, std::enable_if_t<std::is_base_of<To, From>::value>> {
65  static inline bool doit(const From &) { return true; }
66};
67 
68template <typename To, typename From> struct isa_impl_cl {
69  static inline bool doit(const From &Val) {
70    return isa_impl<To, From>::doit(Val);
71  }
72};
73 
74template <typename To, typename From> struct isa_impl_cl<To, const From> {
75  static inline bool doit(const From &Val) {
76    return isa_impl<To, From>::doit(Val);
77  }
78};
79 
80template <typename To, typename From>
81struct isa_impl_cl<To, const std::unique_ptr<From>> {
82  static inline bool doit(const std::unique_ptr<From> &Val) {
83    assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "llvm/include/llvm/Support/Casting.h", 83, __extension__ __PRETTY_FUNCTION__
));
84    return isa_impl_cl<To, From>::doit(*Val);
85  }
86};
87 
88template <typename To, typename From> struct isa_impl_cl<To, From*> {
89  static inline bool doit(const From *Val) {
90    assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "llvm/include/llvm/Support/Casting.h", 90, __extension__ __PRETTY_FUNCTION__
));
91    return isa_impl<To, From>::doit(*Val);
92  }
93};
94 
95template <typename To, typename From> struct isa_impl_cl<To, From*const> {
96  static inline bool doit(const From *Val) {
97    assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "llvm/include/llvm/Support/Casting.h", 97, __extension__ __PRETTY_FUNCTION__
));
98    return isa_impl<To, From>::doit(*Val);
99  }
100};
101 
102template <typename To, typename From> struct isa_impl_cl<To, const From*> {
103  static inline bool doit(const From *Val) {
104    assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "llvm/include/llvm/Support/Casting.h", 104, __extension__ __PRETTY_FUNCTION__
));
105    return isa_impl<To, From>::doit(*Val);
106  }
107};
108 
109template <typename To, typename From> struct isa_impl_cl<To, const From*const> {
110  static inline bool doit(const From *Val) {
111    assert(Val && "isa<> used on a null pointer")(static_cast <bool> (Val && "isa<> used on a null pointer"
) ? void (0) : __assert_fail ("Val && \"isa<> used on a null pointer\""
, "llvm/include/llvm/Support/Casting.h", 111, __extension__ __PRETTY_FUNCTION__
));
112    return isa_impl<To, From>::doit(*Val);
113  }
114};
115 
116template<typename To, typename From, typename SimpleFrom>
117struct isa_impl_wrap {
118  // When From != SimplifiedType, we can simplify the type some more by using
119  // the simplify_type template.
120  static bool doit(const From &Val) {
121    return isa_impl_wrap<To, SimpleFrom,
122      typename simplify_type<SimpleFrom>::SimpleType>::doit(
123                          simplify_type<const From>::getSimplifiedValue(Val));
124  }
125};
126 
127template<typename To, typename FromTy>
128struct isa_impl_wrap<To, FromTy, FromTy> {
129  // When From == SimpleType, we are as simple as we are going to get.
130  static bool doit(const FromTy &Val) {
131    return isa_impl_cl<To,FromTy>::doit(Val);
132  }
133};
134 
135// isa<X> - Return true if the parameter to the template is an instance of one
136// of the template type arguments.  Used like this:
137//
138//  if (isa<Type>(myVal)) { ... }
139//  if (isa<Type0, Type1, Type2>(myVal)) { ... }
140//
141template <class X, class Y> LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
142  return isa_impl_wrap<X, const Y,
143                       typename simplify_type<const Y>::SimpleType>::doit(Val);
144}
145 
146template <typename First, typename Second, typename... Rest, typename Y>
147LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa(const Y &Val) {
148  return isa<First>(Val) || isa<Second, Rest...>(Val);
149}
150 
151// isa_and_nonnull<X> - Functionally identical to isa, except that a null value
152// is accepted.
153//
154template <typename... X, class Y>
155LLVM_NODISCARD[[clang::warn_unused_result]] inline bool isa_and_nonnull(const Y &Val) {
156  if (!Val)
157    return false;
158  return isa<X...>(Val);
159}
160 
161//===----------------------------------------------------------------------===//
162//                          cast<x> Support Templates
163//===----------------------------------------------------------------------===//
164 
165template<class To, class From> struct cast_retty;
166 
167// Calculate what type the 'cast' function should return, based on a requested
168// type of To and a source type of From.
169template<class To, class From> struct cast_retty_impl {
170  using ret_type = To &;       // Normal case, return Ty&
171};
172template<class To, class From> struct cast_retty_impl<To, const From> {
173  using ret_type = const To &; // Normal case, return Ty&
174};
175 
176template<class To, class From> struct cast_retty_impl<To, From*> {
177  using ret_type = To *;       // Pointer arg case, return Ty*
178};
179 
180template<class To, class From> struct cast_retty_impl<To, const From*> {
181  using ret_type = const To *; // Constant pointer arg case, return const Ty*
182};
183 
184template<class To, class From> struct cast_retty_impl<To, const From*const> {
185  using ret_type = const To *; // Constant pointer arg case, return const Ty*
186};
187 
188template <class To, class From>
189struct cast_retty_impl<To, std::unique_ptr<From>> {
190private:
191  using PointerType = typename cast_retty_impl<To, From *>::ret_type;
192  using ResultType = std::remove_pointer_t<PointerType>;
193 
194public:
195  using ret_type = std::unique_ptr<ResultType>;
196};
197 
198template<class To, class From, class SimpleFrom>
199struct cast_retty_wrap {
200  // When the simplified type and the from type are not the same, use the type
201  // simplifier to reduce the type, then reuse cast_retty_impl to get the
202  // resultant type.
203  using ret_type = typename cast_retty<To, SimpleFrom>::ret_type;
204};
205 
206template<class To, class FromTy>
207struct cast_retty_wrap<To, FromTy, FromTy> {
208  // When the simplified type is equal to the from type, use it directly.
209  using ret_type = typename cast_retty_impl<To,FromTy>::ret_type;
210};
211 
212template<class To, class From>
213struct cast_retty {
214  using ret_type = typename cast_retty_wrap<
215      To, From, typename simplify_type<From>::SimpleType>::ret_type;
216};
217 
218// Ensure the non-simple values are converted using the simplify_type template
219// that may be specialized by smart pointers...
220//
221template<class To, class From, class SimpleFrom> struct cast_convert_val {
222  // This is not a simple type, use the template to simplify it...
223  static typename cast_retty<To, From>::ret_type doit(From &Val) {
224    return cast_convert_val<To, SimpleFrom,
23
←
Returning without writing to 'Val.Node'→
225      typename simplify_type<SimpleFrom>::SimpleType>::doit(
226                          simplify_type<From>::getSimplifiedValue(Val));
20
←
Calling 'simplify_type::getSimplifiedValue'→
22
←
Returning from 'simplify_type::getSimplifiedValue'→
227  }
228};
229 
230template<class To, class FromTy> struct cast_convert_val<To,FromTy,FromTy> {
231  // This _is_ a simple type, just cast it.
232  static typename cast_retty<To, FromTy>::ret_type doit(const FromTy &Val) {
233    typename cast_retty<To, FromTy>::ret_type Res2
234     = (typename cast_retty<To, FromTy>::ret_type)const_cast<FromTy&>(Val);
235    return Res2;
236  }
237};
238 
239template <class X> struct is_simple_type {
240  static const bool value =
241      std::is_same<X, typename simplify_type<X>::SimpleType>::value;
242};
243 
244// cast<X> - Return the argument parameter cast to the specified type.  This
245// casting operator asserts that the type is correct, so it does not return null
246// on failure.  It does not allow a null argument (use cast_or_null for that).
247// It is typically used like this:
248//
249//  cast<Instruction>(myVal)->getParent()
250//
251template <class X, class Y>
252inline std::enable_if_t<!is_simple_type<Y>::value,
253                        typename cast_retty<X, const Y>::ret_type>
254cast(const Y &Val) {
255  assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 255, __extension__ __PRETTY_FUNCTION__
));
256  return cast_convert_val<
257      X, const Y, typename simplify_type<const Y>::SimpleType>::doit(Val);
258}
259 
260template <class X, class Y>
261inline typename cast_retty<X, Y>::ret_type cast(Y &Val) {
262  assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 262, __extension__ __PRETTY_FUNCTION__
));
17
←
Assuming 'Val' is a 'ConstantSDNode'→
18
←
'?' condition is true→
263  return cast_convert_val<X, Y,
19
←
Calling 'cast_convert_val::doit'→
24
←
Returning from 'cast_convert_val::doit'→
25
←
Returning without writing to 'Val.Node'→
264                          typename simplify_type<Y>::SimpleType>::doit(Val);
265}
266 
267template <class X, class Y>
268inline typename cast_retty<X, Y *>::ret_type cast(Y *Val) {
269  assert(isa<X>(Val) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 269, __extension__ __PRETTY_FUNCTION__
));
270  return cast_convert_val<X, Y*,
271                          typename simplify_type<Y*>::SimpleType>::doit(Val);
272}
273 
274template <class X, class Y>
275inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
276cast(std::unique_ptr<Y> &&Val) {
277  assert(isa<X>(Val.get()) && "cast<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val.get()) &&
 "cast<Ty>() argument of incompatible type!") ? void (0
) : __assert_fail ("isa<X>(Val.get()) && \"cast<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 277, __extension__ __PRETTY_FUNCTION__
));
278  using ret_type = typename cast_retty<X, std::unique_ptr<Y>>::ret_type;
279  return ret_type(
280      cast_convert_val<X, Y *, typename simplify_type<Y *>::SimpleType>::doit(
281          Val.release()));
282}
283 
284// cast_or_null<X> - Functionally identical to cast, except that a null value is
285// accepted.
286//
287template <class X, class Y>
288LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
289    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
290cast_or_null(const Y &Val) {
291  if (!Val)
292    return nullptr;
293  assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 293, __extension__ __PRETTY_FUNCTION__
));
294  return cast<X>(Val);
295}
296 
297template <class X, class Y>
298LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
299                                       typename cast_retty<X, Y>::ret_type>
300cast_or_null(Y &Val) {
301  if (!Val)
302    return nullptr;
303  assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 303, __extension__ __PRETTY_FUNCTION__
));
304  return cast<X>(Val);
305}
306 
307template <class X, class Y>
308LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
309cast_or_null(Y *Val) {
310  if (!Val) return nullptr;
311  assert(isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!")(static_cast <bool> (isa<X>(Val) && "cast_or_null<Ty>() argument of incompatible type!"
) ? void (0) : __assert_fail ("isa<X>(Val) && \"cast_or_null<Ty>() argument of incompatible type!\""
, "llvm/include/llvm/Support/Casting.h", 311, __extension__ __PRETTY_FUNCTION__
));
312  return cast<X>(Val);
313}
314 
315template <class X, class Y>
316inline typename cast_retty<X, std::unique_ptr<Y>>::ret_type
317cast_or_null(std::unique_ptr<Y> &&Val) {
318  if (!Val)
319    return nullptr;
320  return cast<X>(std::move(Val));
321}
322 
323// dyn_cast<X> - Return the argument parameter cast to the specified type.  This
324// casting operator returns null if the argument is of the wrong type, so it can
325// be used to test for a type as well as cast if successful.  This should be
326// used in the context of an if statement like this:
327//
328//  if (const Instruction *I = dyn_cast<Instruction>(myVal)) { ... }
329//
330 
331template <class X, class Y>
332LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
333    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
334dyn_cast(const Y &Val) {
335  return isa<X>(Val) ? cast<X>(Val) : nullptr;
336}
337 
338template <class X, class Y>
339LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y>::ret_type dyn_cast(Y &Val) {
340  return isa<X>(Val) ? cast<X>(Val) : nullptr;
14
←
Assuming 'Val' is a 'ConstantSDNode'→
15
←
'?' condition is true→
16
←
Calling 'cast<llvm::ConstantSDNode, llvm::SDValue>'→
26
←
Returning from 'cast<llvm::ConstantSDNode, llvm::SDValue>'→
27
←
Returning without writing to 'Val.Node'→
341}
342 
343template <class X, class Y>
344LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type dyn_cast(Y *Val) {
345  return isa<X>(Val) ? cast<X>(Val) : nullptr;
346}
347 
348// dyn_cast_or_null<X> - Functionally identical to dyn_cast, except that a null
349// value is accepted.
350//
351template <class X, class Y>
352LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<
353    !is_simple_type<Y>::value, typename cast_retty<X, const Y>::ret_type>
354dyn_cast_or_null(const Y &Val) {
355  return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
356}
357 
358template <class X, class Y>
359LLVM_NODISCARD[[clang::warn_unused_result]] inline std::enable_if_t<!is_simple_type<Y>::value,
360                                       typename cast_retty<X, Y>::ret_type>
361dyn_cast_or_null(Y &Val) {
362  return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
363}
364 
365template <class X, class Y>
366LLVM_NODISCARD[[clang::warn_unused_result]] inline typename cast_retty<X, Y *>::ret_type
367dyn_cast_or_null(Y *Val) {
368  return (Val && isa<X>(Val)) ? cast<X>(Val) : nullptr;
369}
370 
371// unique_dyn_cast<X> - Given a unique_ptr<Y>, try to return a unique_ptr<X>,
372// taking ownership of the input pointer iff isa<X>(Val) is true.  If the
373// cast is successful, From refers to nullptr on exit and the casted value
374// is returned.  If the cast is unsuccessful, the function returns nullptr
375// and From is unchanged.
376template <class X, class Y>
377LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &Val)
378    -> decltype(cast<X>(Val)) {
379  if (!isa<X>(Val))
380    return nullptr;
381  return cast<X>(std::move(Val));
382}
383 
384template <class X, class Y>
385LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast(std::unique_ptr<Y> &&Val) {
386  return unique_dyn_cast<X, Y>(Val);
387}
388 
389// dyn_cast_or_null<X> - Functionally identical to unique_dyn_cast, except that
390// a null value is accepted.
391template <class X, class Y>
392LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &Val)
393    -> decltype(cast<X>(Val)) {
394  if (!Val)
395    return nullptr;
396  return unique_dyn_cast<X, Y>(Val);
397}
398 
399template <class X, class Y>
400LLVM_NODISCARD[[clang::warn_unused_result]] inline auto unique_dyn_cast_or_null(std::unique_ptr<Y> &&Val) {
401  return unique_dyn_cast_or_null<X, Y>(Val);
402}
403 
404} // end namespace llvm
405 
406#endif // LLVM_SUPPORT_CASTING_H

←

/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>

57namespace llvm {

59class APInt;
60class Constant;
61class GlobalValue;
62class MachineBasicBlock;
63class MachineConstantPoolValue;
64class MCSymbol;
65class raw_ostream;
66class SDNode;
67class SelectionDAG;
68class Type;
69class Value;

71void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

74/// This represents a list of ValueType's that has been intern'd by
75/// a SelectionDAG.  Instances of this simple value class are returned by
76/// SelectionDAG::getVTList(...).
77///
78struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
81};

83namespace ISD {

/// Node predicates

87/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
88/// same constant or undefined, return true and return the constant value in
89/// \p SplatValue.
90bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

92/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
93/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
94/// true, it only checks BUILD_VECTOR.
95bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

98/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
99/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
100/// only checks BUILD_VECTOR.
101bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

104/// Return true if the specified node is a BUILD_VECTOR where all of the
105/// elements are ~0 or undef.
106bool isBuildVectorAllOnes(const SDNode *N);

108/// Return true if the specified node is a BUILD_VECTOR where all of the
109/// elements are 0 or undef.
110bool isBuildVectorAllZeros(const SDNode *N);

112/// Return true if the specified node is a BUILD_VECTOR node of all
113/// ConstantSDNode or undef.
114bool isBuildVectorOfConstantSDNodes(const SDNode *N);

116/// Return true if the specified node is a BUILD_VECTOR node of all
117/// ConstantFPSDNode or undef.
118bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

120/// Return true if the node has at least one operand and all operands of the
121/// specified node are ISD::UNDEF.
122bool allOperandsUndef(const SDNode *N);

124} // end namespace ISD

126//===----------------------------------------------------------------------===//
127/// Unlike LLVM values, Selection DAG nodes may return multiple
128/// values as the result of a computation.  Many nodes return multiple values,
129/// from loads (which define a token and a return value) to ADDC (which returns
130/// a result and a carry value), to calls (which may return an arbitrary number
131/// of values).
132///
133/// As such, each use of a SelectionDAG computation must indicate the node that
134/// computes it as well as which return value to use from that node.  This pair
135/// of information is represented with the SDValue value type.
136///
137class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
unsigned ResNo = 0;     // Which return value of the node we are using.

143public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
229};

231template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
252};

254/// Allow casting operators to work directly on
255/// SDValues as if they were SDNode*'s.
256template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
21
←
Returning without writing to 'Val.Node'→
}
262};
263template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
269};

271/// Represents a use of a SDNode. This class holds an SDValue,
272/// which records the SDNode being used and the result number, a
273/// pointer to the SDNode using the value, and Next and Prev pointers,
274/// which link together all the uses of an SDNode.
275///
276class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

286public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

326private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
355};

357/// simplify_type specializations - Allow casting operators to work directly on
358/// SDValues as if they were SDNode*'s.
359template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
365};

367/// These are IR-level optimization flags that may be propagated to SDNodes.
368/// TODO: This data structure should be shared by the IR optimizer and the
369/// the backend.
370struct SDNodeFlags {
371private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

390public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
450};

452/// Represents one node in the SelectionDAG.
453///
454class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
455private:
/// The operation that this node performs.
int16_t NodeType;

459protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with NodeType.

464#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
465// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
466// and give the `pack` pragma push semantics.
467#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
468#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
469#else
470#define BEGIN_TWO_BYTE_PACK()
471#define END_TWO_BYTE_PACK()
472#endif

474BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class VPBaseLoadStoreSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;
  friend class VPGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   VPGatherScatterSDNode => enum ISD::MemIndexType
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class VPLoadSDNode;
  friend class VPStridedLoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;
  friend class VPGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class VPStoreSDNode;
  friend class VPStridedStoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;
  friend class VPScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
566END_TWO_BYTE_PACK()
567#undef BEGIN_TWO_BYTE_PACK
568#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

580private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

617public:
/// Unique and persistent id per SDNode in the DAG. Used for debug printing.
/// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS`
/// intentionally because it adds unneeded complexity without noticeable
/// benefits (see discussion with @thakis in D120714).
uint16_t PersistentId;

//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned short)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
675#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
677#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node is a vector predication operation.
bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 693, __extension__
 __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using iterator_category = std::forward_iterator_tag;
  using value_type = SDUse;
  using difference_type = std::ptrdiff_t;
  using pointer = value_type *;
  using reference = value_type &;

  use_iterator() = default;
  use_iterator(const use_iterator &I) = default;

  bool operator==(const use_iterator &x) const { return Op == x.Op; }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 760, __extension__
 __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 771, __extension__
 __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 781, __extension__
 __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 909, __extension__
 __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 971, __extension__
 __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1058protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
 "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1072, __extension__
 __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1074, __extension__
 __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1074, __extension__
 __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1079};

1081/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1082/// into SDNode creation functions.
1083/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1084/// from the original Instruction, and IROrder is the ordinal position of
1085/// the instruction.
1086/// When an SDNode is created after the DAG is being built, both DebugLoc and
1087/// the IROrder are propagated from the original SDNode.
1088/// So SDLoc class provides two constructors besides the default one, one to
1089/// be used by the DAGBuilder, the other to be used by others.
1090class SDLoc {
1091private:
DebugLoc DL;
int IROrder = 0;

1095public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1100, __extension__
 __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1107};

1109// Define inline functions from the SDValue class.

1111inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1117, __extension__
 __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1117, __extension__
 __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1118, __extension__
 __PRETTY_FUNCTION__));
1119}

1121inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
1123}

1125inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
1127}

1129inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1131}

1133inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1135}

1137inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1139}

1141inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1143}

1145inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1147}

1149inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1151}

1153inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1155}

1157inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1159}

1161inline bool SDValue::isUndef() const {
return Node->isUndef();
1163}

1165inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1167}

1169inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
32
←
Called C++ object pointer is null
1171}

1173inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1175}

1177inline void SDValue::dump() const {
return Node->dump();
1179}

1181inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1183}

1185inline void SDValue::dumpr() const {
return Node->dumpr();
1187}

1189inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1191}

1193// Define inline functions from the SDUse class.

1195inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode())
  V->addUse(*this);
1200}

1202inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V->addUse(*this);
1205}

1207inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1211}

1213/// This class is used to form a handle around another node that
1214/// is persistent and is updated across invocations of replaceAllUsesWith on its
1215/// operand.  This node should be directly created by end-users and not added to
1216/// the AllNodes list.
1217class HandleSDNode : public SDNode {
SDUse Op;

1220public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1239};

1241class AddrSpaceCastSDNode : public SDNode {
1242private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1246public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1256};

1258/// This is an abstract virtual class for memory operations.
1259class MemSDNode : public SDNode {
1260private:
// VT of in-memory value.
EVT MemoryVT;

1264protected:
/// Memory reference information.
MachineMemOperand *MMO;

1268public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }
// FIXME: Remove once transition to getAlign is over.
unsigned getAlignment() const { return MMO->getAlign().value(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getSuccessOrdering() const {
  return MMO->getSuccessOrdering();
}

/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation.  (For operations
/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::VP_STORE:
  case ISD::MSTORE:
  case ISD::VP_SCATTER:
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  switch (N->getOpcode()) {
  case ISD::LOAD:
  case ISD::STORE:
  case ISD::PREFETCH:
  case ISD::ATOMIC_CMP_SWAP:
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  case ISD::ATOMIC_SWAP:
  case ISD::ATOMIC_LOAD_ADD:
  case ISD::ATOMIC_LOAD_SUB:
  case ISD::ATOMIC_LOAD_AND:
  case ISD::ATOMIC_LOAD_CLR:
  case ISD::ATOMIC_LOAD_OR:
  case ISD::ATOMIC_LOAD_XOR:
  case ISD::ATOMIC_LOAD_NAND:
  case ISD::ATOMIC_LOAD_MIN:
  case ISD::ATOMIC_LOAD_MAX:
  case ISD::ATOMIC_LOAD_UMIN:
  case ISD::ATOMIC_LOAD_UMAX:
  case ISD::ATOMIC_LOAD_FADD:
  case ISD::ATOMIC_LOAD_FSUB:
  case ISD::ATOMIC_LOAD:
  case ISD::ATOMIC_STORE:
  case ISD::MLOAD:
  case ISD::MSTORE:
  case ISD::MGATHER:
  case ISD::MSCATTER:
  case ISD::VP_LOAD:
  case ISD::VP_STORE:
  case ISD::VP_GATHER:
  case ISD::VP_SCATTER:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return true;
  default:
    return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
  }
}
1423};

1425/// This is an SDNode representing atomic operations.
1426class AtomicSDNode : public MemSDNode {
1427public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1432, __extension__
 __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1432, __extension__
 __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1449, __extension__
 __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1474};

1476/// This SDNode is used for target intrinsics that touch
1477/// memory and need an associated MachineMemOperand. Its opcode may be
1478/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1479/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1480class MemIntrinsicSDNode : public MemSDNode {
1481public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1496};

1498/// This SDNode is used to implement the code generator
1499/// support for the llvm IR shufflevector instruction.  It combines elements
1500/// from two input vectors into a new input vector, with the selection and
1501/// ordering of elements determined by an array of integers, referred to as
1502/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1503/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1504/// An index of -1 is treated as undef, such that the code generator may put
1505/// any value in the corresponding element of the result.
1506class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1511protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1517public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return makeArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
 ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1524, __extension__
 __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1531, __extension__
 __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1562};

1564class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1576public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isZero() const { return Value->isZero(); }
// NOTE: This is soft-deprecated.  Please use `isZero()` instead.
bool isNullValue() const { return isZero(); }
bool isAllOnes() const { return Value->isMinusOne(); }
// NOTE: This is soft-deprecated.  Please use `isAllOnes()` instead.
bool isAllOnesValue() const { return isAllOnes(); }
bool isMaxSignedValue() const { return Value->isMaxValue(true); }
bool isMinSignedValue() const { return Value->isMinValue(true); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1603};

1605uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1607}

1609const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1611}

1613class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1623public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1658};

1660/// Returns true if \p V is a constant integer zero.
1661bool isNullConstant(SDValue V);

1663/// Returns true if \p V is an FP constant with a value of positive zero.
1664bool isNullFPConstant(SDValue V);

1666/// Returns true if \p V is an integer constant with all bits set.
1667bool isAllOnesConstant(SDValue V);

1669/// Returns true if \p V is a constant integer one.
1670bool isOneConstant(SDValue V);

1672/// Returns true if \p V is a constant min signed integer value.
1673bool isMinSignedConstant(SDValue V);

1675/// Return the non-bitcasted source operand of \p V if it exists.
1676/// If \p V is not a bitcasted value, it is returned as-is.
1677SDValue peekThroughBitcasts(SDValue V);

1679/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1680/// If \p V is not a bitcasted one-use value, it is returned as-is.
1681SDValue peekThroughOneUseBitcasts(SDValue V);

1683/// Return the non-extracted vector source operand of \p V if it exists.
1684/// If \p V is not an extracted subvector, it is returned as-is.
1685SDValue peekThroughExtractSubvectors(SDValue V);

1687/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1688/// constant is canonicalized to be operand 1.
1689bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1691/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1692ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1695/// Returns the SDNode if it is a demanded constant splat BuildVector or
1696/// constant int.
1697ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1701/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1702ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1704/// Returns the SDNode if it is a demanded constant splat BuildVector or
1705/// constant float.
1706ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1709/// Return true if the value is a constant 0 integer or a splatted vector of
1710/// a constant 0 integer (with no undefs by default).
1711/// Build vector implicit truncation is not an issue for null values.
1712bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1714/// Return true if the value is a constant 1 integer or a splatted vector of a
1715/// constant 1 integer (with no undefs).
1716/// Does not permit build vector implicit truncation.
1717bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1719/// Return true if the value is a constant -1 integer or a splatted vector of a
1720/// constant -1 integer (with no undefs).
1721/// Does not permit build vector implicit truncation.
1722bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1724/// Return true if \p V is either a integer or FP constant.
1725inline bool isIntOrFPConstant(SDValue V) {
return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1727}

1729class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1740public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1753};

1755class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1765public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1772};

1774/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1775/// the offet and size that are started/ended in the underlying FrameIndex.
1776class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1784public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1791, __extension__
 __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1795, __extension__
 __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1804};

1806/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1807/// the index of the basic block being probed. A pseudo probe serves as a place
1808/// holder and will be removed at the end of compilation. It does not have any
1809/// operand because we do not want the instruction selection to deal with any.
1810class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1821public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1830};

1832class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1843public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1851};

1853class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1869, __extension__
 __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1878, __extension__
 __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1883public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1889, __extension__
 __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1894, __extension__
 __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1913};

1915/// Completely target-dependent object reference.
1916class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1923public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1935};

1937class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1949public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1955};

1957/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1958class BuildVectorSDNode : public SDNode {
1959public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

/// Extract the raw bit data from a build vector of Undef, Constant or
/// ConstantFP node elements. Each raw bit element will be \p
/// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
/// undefined elements are flagged in \p UndefElements.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                        SmallVectorImpl<APInt> &RawBitElements,
                        BitVector &UndefElements) const;

bool isConstant() const;

/// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements.
/// Undef elements are treated as zero, and entirely undefined elements are
/// flagged in \p DstUndefElements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                          SmallVectorImpl<APInt> &DstBitElements,
                          ArrayRef<APInt> SrcBitElements,
                          BitVector &DstUndefElements,
                          const BitVector &SrcUndefElements);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2085};

2087/// An SDNode that holds an arbitrary LLVM IR Value. This is
2088/// used when the SelectionDAG needs to make a simple reference to something
2089/// in the LLVM IR representation.
2090///
2091class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2100public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2107};

2109class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2118public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2124};

2126class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2134public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2140};

2142class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2152public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2158};

2160class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2172public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2181};

2183class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
 "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2190, __extension__
 __PRETTY_FUNCTION__));
}

2193public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2200};

2202class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2213public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2221};

2223class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2231public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2237};

2239class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2248public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2254};

2256/// This class is used to represent EVT's, which are used
2257/// to parameterize some operations.
2258class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2267public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2273};

2275/// Base class for LoadSDNode and StoreSDNode
2276class LSBaseSDNode : public MemSDNode {
2277public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2283, __extension__
 __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2306};

2308/// This class is used to represent ISD::LOAD nodes.
2309class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2317, __extension__
 __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2318, __extension__
 __PRETTY_FUNCTION__));
}

2321public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2334};

2336/// This class is used to represent ISD::STORE nodes.
2337class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2345, __extension__
 __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2346, __extension__
 __PRETTY_FUNCTION__));
}

2349public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2365};

2367/// This base class is used to represent VP_LOAD, VP_STORE,
2368/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes
2369class VPBaseLoadStoreSDNode : public MemSDNode {
2370public:
friend class SelectionDAG;

VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &DL, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2379, __extension__
 __PRETTY_FUNCTION__));
}

// VPStridedStoreSDNode (Chain, Data, Ptr,    Offset, Stride, Mask, EVL)
// VPStoreSDNode        (Chain, Data, Ptr,    Offset, Mask,   EVL)
// VPStridedLoadSDNode  (Chain, Ptr,  Offset, Stride, Mask,   EVL)
// VPLoadSDNode         (Chain, Ptr,  Offset, Mask,   EVL)
// Mask is a vector of i1 elements;
// the type of EVL is TLI.getVPExplicitVectorLengthTy().
const SDValue &getOffset() const {
  return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
                     getOpcode() == ISD::VP_LOAD)
                        ? 2
                        : 3);
}
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
                     getOpcode() == ISD::VP_LOAD)
                        ? 1
                        : 2);
}
const SDValue &getMask() const {
  switch (getOpcode()) {
  default:
    llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2403);
  case ISD::VP_LOAD:
    return getOperand(3);
  case ISD::VP_STORE:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return getOperand(4);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(5);
  }
}
const SDValue &getVectorLength() const {
  switch (getOpcode()) {
  default:
    llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2416);
  case ISD::VP_LOAD:
    return getOperand(4);
  case ISD::VP_STORE:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return getOperand(5);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(6);
  }
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
         N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE ||
         N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
}
2444};

2446/// This class is used to represent a VP_LOAD node
2447class VPLoadSDNode : public VPBaseLoadStoreSDNode {
2448public:
friend class SelectionDAG;

VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
             ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
             EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = isExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getVectorLength() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2472};

2474/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
2475class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode {
2476public:
friend class SelectionDAG;

VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
                    ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                    bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs,
                            AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getStride() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2502};

2504/// This class is used to represent a VP_STORE node
2505class VPStoreSDNode : public VPBaseLoadStoreSDNode {
2506public:
friend class SelectionDAG;

VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
              ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
              EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_STORE;
}
2537};

2539/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
2540class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode {
2541public:
friend class SelectionDAG;

VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
                     ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing,
                     EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL,
                            VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
  StoreSDNodeBits.IsCompressing = IsCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getStride() const { return getOperand(4); }
const SDValue &getMask() const { return getOperand(5); }
const SDValue &getVectorLength() const { return getOperand(6); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE;
}
2574};

2576/// This base class is used to represent MLOAD and MSTORE nodes
2577class MaskedLoadStoreSDNode : public MemSDNode {
2578public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2587, __extension__
 __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2616};

2618/// This class is used to represent an MLOAD node
2619class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2620public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2645};

2647/// This class is used to represent an MSTORE node
2648class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2649public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2679};

2681/// This is a base class used to represent
2682/// VP_GATHER and VP_SCATTER nodes
2683///
2684class VPGatherScatterSDNode : public MemSDNode {
2685public:
friend class SelectionDAG;

VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                      MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2693, __extension__
 __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// VPGatherSDNode  (Chain, base, index, scale, mask, vlen)
// VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
}
const SDValue &getIndex() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
}
const SDValue &getScale() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
}
const SDValue &getMask() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
}
const SDValue &getVectorLength() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER ||
         N->getOpcode() == ISD::VP_SCATTER;
}
2733};

2735/// This class is used to represent an VP_GATHER node
2736///
2737class VPGatherSDNode : public VPGatherScatterSDNode {
2738public:
friend class SelectionDAG;

VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
               MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER;
}
2749};

2751/// This class is used to represent an VP_SCATTER node
2752///
2753class VPScatterSDNode : public VPGatherScatterSDNode {
2754public:
friend class SelectionDAG;

VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_SCATTER;
}
2767};

2769/// This is a base class used to represent
2770/// MGATHER and MSCATTER nodes
2771///
2772class MaskedGatherScatterSDNode : public MemSDNode {
2773public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2781, __extension__
 __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
void setIndexType(ISD::MemIndexType IndexType) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2813};

2815/// This class is used to represent an MGATHER node
2816///
2817class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2818public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2838};

2840/// This class is used to represent an MSCATTER node
2841///
2842class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2843public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2864};

2866/// An SDNode that represents everything that will be needed
2867/// to construct a MachineInstr. These nodes are created during the
2868/// instruction selection proper phase.
2869///
2870/// Note that the only supported way to set the `memoperands` is by calling the
2871/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2872/// inside the DAG rather than in the node.
2873class MachineSDNode : public SDNode {
2874private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2902public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2928};

2930/// An SDNode that records if a register contains a value that is guaranteed to
2931/// be aligned accordingly.
2932class AssertAlignSDNode : public SDNode {
Align Alignment;

2935public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2944};

2946class SDNodeIterator {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2952public:
using iterator_category = std::forward_iterator_tag;
using value_type = SDNode;
using difference_type = std::ptrdiff_t;
using pointer = value_type *;
using reference = value_type &;

bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2978, __extension__
 __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2978, __extension__
 __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
2989};

2991template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
3004};

3006/// A representation of the largest SDNode, for use in sizeof().
3007///
3008/// This needs to be a union because the largest node differs on 32 bit systems
3009/// with 4 and 8 byte pointer alignment, respectively.
3010using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

3015/// The SDNode class with the greatest alignment requirement.
3016using MostAlignedSDNode = GlobalAddressSDNode;

3018namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

3096} // end namespace ISD

3098} // end namespace llvm

3100#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H