/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Bug Summary

File:	build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1126, column 10 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name LegalizeIntegerTypes.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm -resource-dir /usr/lib/llvm-15/lib/clang/15.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-15/lib/clang/15.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-04-20-140412-16051-1 -x c++ /build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

→

1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type.  For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type.  For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//

20#include "LegalizeTypes.h"
21#include "llvm/Analysis/TargetLibraryInfo.h"
22#include "llvm/IR/DerivedTypes.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/KnownBits.h"
25#include "llvm/Support/raw_ostream.h"
26#include <algorithm>
27using namespace llvm;

29#define DEBUG_TYPE"legalize-types" "legalize-types"

31//===----------------------------------------------------------------------===//
32//  Integer Result Promotion
33//===----------------------------------------------------------------------===//

35/// PromoteIntegerResult - This method is called when a result of a node is
36/// found to be in need of promotion to a larger type.  At this point, the node
37/// may also have invalid operands or may have other results that need
38/// expansion, we just know that (at least) one result needs promotion.
39void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
           dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
);
SDValue Res = SDValue();

// See if the target wants to custom expand this node.
if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Node has been custom expanded, done\n"
; } } while (false);
  return;
}

switch (N->getOpcode()) {
default:
52#ifndef NDEBUG
  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
  N->dump(&DAG); dbgs() << "\n";
55#endif
  llvm_unreachable("Do not know how to promote this operator!")::llvm::llvm_unreachable_internal("Do not know how to promote this operator!"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 56
);
case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
case ISD::AssertSext:  Res = PromoteIntRes_AssertSext(N); break;
case ISD::AssertZext:  Res = PromoteIntRes_AssertZext(N); break;
case ISD::BITCAST:     Res = PromoteIntRes_BITCAST(N); break;
case ISD::BITREVERSE:  Res = PromoteIntRes_BITREVERSE(N); break;
case ISD::BSWAP:       Res = PromoteIntRes_BSWAP(N); break;
case ISD::BUILD_PAIR:  Res = PromoteIntRes_BUILD_PAIR(N); break;
case ISD::Constant:    Res = PromoteIntRes_Constant(N); break;
case ISD::CTLZ_ZERO_UNDEF:
case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
case ISD::PARITY:
case ISD::CTPOP:       Res = PromoteIntRes_CTPOP_PARITY(N); break;
case ISD::CTTZ_ZERO_UNDEF:
case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
case ISD::EXTRACT_VECTOR_ELT:
                       Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
case ISD::MLOAD:       Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
  break;
case ISD::MGATHER:     Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
  break;
case ISD::SELECT:
case ISD::VSELECT:
case ISD::VP_SELECT:
  Res = PromoteIntRes_Select(N);
  break;
case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS:
case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
case ISD::SMIN:
case ISD::SMAX:        Res = PromoteIntRes_SExtIntBinOp(N); break;
case ISD::UMIN:
case ISD::UMAX:        Res = PromoteIntRes_UMINUMAX(N); break;

case ISD::SHL:
case ISD::VP_SHL:      Res = PromoteIntRes_SHL(N); break;
case ISD::SIGN_EXTEND_INREG:
                       Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
case ISD::SRA:
case ISD::VP_ASHR:     Res = PromoteIntRes_SRA(N); break;
case ISD::SRL:
case ISD::VP_LSHR:     Res = PromoteIntRes_SRL(N); break;
case ISD::TRUNCATE:    Res = PromoteIntRes_TRUNCATE(N); break;
case ISD::UNDEF:       Res = PromoteIntRes_UNDEF(N); break;
case ISD::VAARG:       Res = PromoteIntRes_VAARG(N); break;
case ISD::VSCALE:      Res = PromoteIntRes_VSCALE(N); break;

case ISD::EXTRACT_SUBVECTOR:
                       Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
case ISD::INSERT_SUBVECTOR:
                       Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
case ISD::VECTOR_REVERSE:
                       Res = PromoteIntRes_VECTOR_REVERSE(N); break;
case ISD::VECTOR_SHUFFLE:
                       Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
case ISD::VECTOR_SPLICE:
                       Res = PromoteIntRes_VECTOR_SPLICE(N); break;
case ISD::INSERT_VECTOR_ELT:
                       Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
case ISD::BUILD_VECTOR:
                       Res = PromoteIntRes_BUILD_VECTOR(N); break;
case ISD::SCALAR_TO_VECTOR:
                       Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
case ISD::SPLAT_VECTOR:
                       Res = PromoteIntRes_SPLAT_VECTOR(N); break;
case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
case ISD::CONCAT_VECTORS:
                       Res = PromoteIntRes_CONCAT_VECTORS(N); break;

case ISD::ANY_EXTEND_VECTOR_INREG:
case ISD::SIGN_EXTEND_VECTOR_INREG:
case ISD::ZERO_EXTEND_VECTOR_INREG:
                       Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;

case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
case ISD::ANY_EXTEND:  Res = PromoteIntRes_INT_EXTEND(N); break;

case ISD::STRICT_FP_TO_SINT:
case ISD::STRICT_FP_TO_UINT:
case ISD::FP_TO_SINT:
case ISD::FP_TO_UINT:  Res = PromoteIntRes_FP_TO_XINT(N); break;

case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT:
                       Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;

case ISD::FP_TO_FP16:  Res = PromoteIntRes_FP_TO_FP16(N); break;

case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;

case ISD::AND:
case ISD::OR:
case ISD::XOR:
case ISD::ADD:
case ISD::SUB:
case ISD::MUL:
case ISD::VP_AND:
case ISD::VP_OR:
case ISD::VP_XOR:
case ISD::VP_ADD:
case ISD::VP_SUB:
case ISD::VP_MUL:      Res = PromoteIntRes_SimpleIntBinOp(N); break;

case ISD::SDIV:
case ISD::SREM:
case ISD::VP_SDIV:
case ISD::VP_SREM:     Res = PromoteIntRes_SExtIntBinOp(N); break;

case ISD::UDIV:
case ISD::UREM:
case ISD::VP_UDIV:
case ISD::VP_UREM:     Res = PromoteIntRes_ZExtIntBinOp(N); break;

case ISD::SADDO:
case ISD::SSUBO:       Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
case ISD::UADDO:
case ISD::USUBO:       Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
case ISD::SMULO:
case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;

case ISD::ADDE:
case ISD::SUBE:
case ISD::ADDCARRY:
case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;

case ISD::SADDO_CARRY:
case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;

case ISD::SADDSAT:
case ISD::UADDSAT:
case ISD::SSUBSAT:
case ISD::USUBSAT:
case ISD::SSHLSAT:
case ISD::USHLSAT:     Res = PromoteIntRes_ADDSUBSHLSAT(N); break;

case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
case ISD::UMULFIXSAT:  Res = PromoteIntRes_MULFIX(N); break;

case ISD::SDIVFIX:
case ISD::SDIVFIXSAT:
case ISD::UDIVFIX:
case ISD::UDIVFIXSAT:  Res = PromoteIntRes_DIVFIX(N); break;

case ISD::ABS:         Res = PromoteIntRes_ABS(N); break;

case ISD::ATOMIC_LOAD:
  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;

case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
case ISD::ATOMIC_LOAD_MIN:
case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX:
case ISD::ATOMIC_SWAP:
  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;

case ISD::ATOMIC_CMP_SWAP:
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
  break;

case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN:
  Res = PromoteIntRes_VECREDUCE(N);
  break;

case ISD::VP_REDUCE_ADD:
case ISD::VP_REDUCE_MUL:
case ISD::VP_REDUCE_AND:
case ISD::VP_REDUCE_OR:
case ISD::VP_REDUCE_XOR:
case ISD::VP_REDUCE_SMAX:
case ISD::VP_REDUCE_SMIN:
case ISD::VP_REDUCE_UMAX:
case ISD::VP_REDUCE_UMIN:
  Res = PromoteIntRes_VP_REDUCE(N);
  break;

case ISD::FREEZE:
  Res = PromoteIntRes_FREEZE(N);
  break;

case ISD::ROTL:
case ISD::ROTR:
  Res = PromoteIntRes_Rotate(N);
  break;

case ISD::FSHL:
case ISD::FSHR:
  Res = PromoteIntRes_FunnelShift(N);
  break;
}

// If the result is null then the sub-method took care of registering it.
if (Res.getNode())
  SetPromotedInteger(SDValue(N, ResNo), Res);
270}

272SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
                                                   unsigned ResNo) {
SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
return GetPromotedInteger(Op);
276}

278SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
// Sign-extend the new bits, and continue the assertion.
SDValue Op = SExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::AssertSext, SDLoc(N),
                   Op.getValueType(), Op, N->getOperand(1));
283}

285SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
// Zero the new bits, and continue the assertion.
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::AssertZext, SDLoc(N),
                   Op.getValueType(), Op, N->getOperand(1));
290}

292SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
                            N->getMemoryVT(), ResVT,
                            N->getChain(), N->getBasePtr(),
                            N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
302}

304SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
                            N->getMemoryVT(),
                            N->getChain(), N->getBasePtr(),
                            Op2, N->getMemOperand());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
314}

316SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
                                                    unsigned ResNo) {
if (ResNo == 1) {
  assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS)(static_cast <bool> (N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 319
, __extension__ __PRETTY_FUNCTION__));
  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));

  // Only use the result of getSetCCResultType if it is legal,
  // otherwise just use the promoted result type (NVT).
  if (!TLI.isTypeLegal(SVT))
    SVT = NVT;

  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
  SDValue Res = DAG.getAtomicCmpSwap(
      ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
      N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
      N->getMemOperand());
  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
  return Res.getValue(1);
}

// Op2 is used for the comparison and thus must be extended according to the
// target's atomic operations. Op3 is merely stored and so can be left alone.
SDValue Op2 = N->getOperand(2);
SDValue Op3 = GetPromotedInteger(N->getOperand(3));
switch (TLI.getExtendForAtomicCmpSwapArg()) {
case ISD::SIGN_EXTEND:
  Op2 = SExtPromotedInteger(Op2);
  break;
case ISD::ZERO_EXTEND:
  Op2 = ZExtPromotedInteger(Op2);
  break;
case ISD::ANY_EXTEND:
  Op2 = GetPromotedInteger(Op2);
  break;
default:
  llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 353
);
}

SDVTList VTs =
    DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
SDValue Res = DAG.getAtomicCmpSwap(
    N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
    N->getBasePtr(), Op2, Op3, N->getMemOperand());
// Update the use to N with the newly created Res.
for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
return Res;
365}

367SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
SDValue InOp = N->getOperand(0);
EVT InVT = InOp.getValueType();
EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
SDLoc dl(N);

switch (getTypeAction(InVT)) {
case TargetLowering::TypeLegal:
  break;
case TargetLowering::TypePromoteInteger:
  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
    // The input promotes to the same size.  Convert the promoted value.
    return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
  break;
case TargetLowering::TypeSoftenFloat:
  // Promote the integer operand by hand.
  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
case TargetLowering::TypeSoftPromoteHalf:
  // Promote the integer operand by hand.
  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
case TargetLowering::TypePromoteFloat: {
  // Convert the promoted float by hand.
  if (!NOutVT.isVector())
    return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
  break;
}
case TargetLowering::TypeExpandInteger:
case TargetLowering::TypeExpandFloat:
  break;
case TargetLowering::TypeScalarizeVector:
  // Convert the element to an integer and promote it by hand.
  if (!NOutVT.isVector())
    return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
                       BitConvertToInteger(GetScalarizedVector(InOp)));
  break;
case TargetLowering::TypeScalarizeScalableVector:
  report_fatal_error("Scalarization of scalable vectors is not supported.");
case TargetLowering::TypeSplitVector: {
  if (!NOutVT.isVector()) {
    // For example, i32 = BITCAST v2i16 on alpha.  Convert the split
    // pieces of the input into integers and reassemble in the final type.
    SDValue Lo, Hi;
    GetSplitVector(N->getOperand(0), Lo, Hi);
    Lo = BitConvertToInteger(Lo);
    Hi = BitConvertToInteger(Hi);

    if (DAG.getDataLayout().isBigEndian())
      std::swap(Lo, Hi);

    InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
                       EVT::getIntegerVT(*DAG.getContext(),
                                         NOutVT.getSizeInBits()),
                       JoinIntegers(Lo, Hi));
    return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
  }
  break;
}
case TargetLowering::TypeWidenVector:
  // The input is widened to the same size. Convert to the widened value.
  // Make sure that the outgoing value is not a vector, because this would
  // make us bitcast between two vectors which are legalized in different ways.
  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
    SDValue Res =
      DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));

    // For big endian targets we need to shift the casted value or the
    // interesting bits will end up at the wrong place.
    if (DAG.getDataLayout().isBigEndian()) {
      unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
      EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
      assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!")(static_cast <bool> (ShiftAmt < NOutVT.getSizeInBits
() && "Too large shift amount!") ? void (0) : __assert_fail
 ("ShiftAmt < NOutVT.getSizeInBits() && \"Too large shift amount!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 439
, __extension__ __PRETTY_FUNCTION__));
      Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
                        DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
    }
    return Res;
  }
  // If the output type is also a vector and widening it to the same size
  // as the widened input type would be a legal type, we can widen the bitcast
  // and handle the promotion after.
  if (NOutVT.isVector()) {
    unsigned WidenInSize = NInVT.getSizeInBits();
    unsigned OutSize = OutVT.getSizeInBits();
    if (WidenInSize % OutSize == 0) {
      unsigned Scale = WidenInSize / OutSize;
      EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
                                       OutVT.getVectorElementType(),
                                       OutVT.getVectorNumElements() * Scale);
      if (isTypeLegal(WideOutVT)) {
        InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
        InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
                           DAG.getVectorIdxConstant(0, dl));
        return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
      }
    }
  }
}

return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
                   CreateStackStoreLoad(InOp, OutVT));
468}

470SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
SDValue V = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::FREEZE, SDLoc(N),
                   V.getValueType(), V);
474}

476SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
EVT OVT = N->getValueType(0);
EVT NVT = Op.getValueType();
SDLoc dl(N);

// If the larger BSWAP isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type. We only do this for scalars since we have a shuffle
// based lowering for vectors in LegalizeVectorOps.
if (!OVT.isVector() &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::BSWAP, NVT)) {
  if (SDValue Res = TLI.expandBSWAP(N, DAG))
    return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
}

unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
EVT ShiftVT = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
                   DAG.getConstant(DiffBits, dl, ShiftVT));
496}

498SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
EVT OVT = N->getValueType(0);
EVT NVT = Op.getValueType();
SDLoc dl(N);

// If the larger BITREVERSE isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type. We only do this for scalars since we have a shuffle
// based lowering for vectors in LegalizeVectorOps.
if (!OVT.isVector() && OVT.isSimple() &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::BITREVERSE, NVT)) {
  if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
    return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
}

unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
EVT ShiftVT = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
return DAG.getNode(ISD::SRL, dl, NVT,
                   DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
                   DAG.getConstant(DiffBits, dl, ShiftVT));
519}

521SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
// The pair element type may be legal, or may not promote to the same type as
// the result, for example i14 = BUILD_PAIR (i7, i7).  Handle all cases.
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
                   TLI.getTypeToTransformTo(*DAG.getContext(),
                   N->getValueType(0)), JoinIntegers(N->getOperand(0),
                   N->getOperand(1)));
528}

530SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
EVT VT = N->getValueType(0);
// FIXME there is no actual debug info here
SDLoc dl(N);
// Zero extend things like i1, sign extend everything else.  It shouldn't
// matter in theory which one we pick, but this tends to give better code?
unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
SDValue Result = DAG.getNode(Opc, dl,
                             TLI.getTypeToTransformTo(*DAG.getContext(), VT),
                             SDValue(N, 0));
assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?")(static_cast <bool> (isa<ConstantSDNode>(Result) &&
 "Didn't constant fold ext?") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Result) && \"Didn't constant fold ext?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 540
, __extension__ __PRETTY_FUNCTION__));
return Result;
542}

544SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
SDLoc dl(N);

// If the larger CTLZ isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type.
if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ, NVT) &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::CTLZ_ZERO_UNDEF, NVT)) {
  if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
    Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
    return Result;
  }
}

// Zero extend to the promoted type and do the count there.
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
// Subtract off the extra leading bits in the bigger type.
return DAG.getNode(
    ISD::SUB, dl, NVT, Op,
    DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
                    NVT));
569}

571SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);

// If the larger CTPOP isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type.
// TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
// TargetLowering.
if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::CTPOP, NVT)) {
  if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
    Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
    return Result;
  }
}

// Zero extend to the promoted type and do the count or parity there.
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
591}

593SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
EVT OVT = N->getValueType(0);
EVT NVT = Op.getValueType();
SDLoc dl(N);

// If the larger CTTZ isn't supported by the target, try to expand now.
// If we expand later we'll end up with more operations since we lost the
// original type. Don't expand if we can use CTPOP or CTLZ expansion on the
// larger type.
if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ, NVT) &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::CTTZ_ZERO_UNDEF, NVT) &&
    !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
    !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
  if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
    Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
    return Result;
  }
}

if (N->getOpcode() == ISD::CTTZ) {
  // The count is the same in the promoted type except if the original
  // value was zero.  This can be handled by setting the bit just off
  // the top of the original type.
  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
                                    OVT.getScalarSizeInBits());
  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
}
return DAG.getNode(N->getOpcode(), dl, NVT, Op);
623}

625SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
SDLoc dl(N);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));

SDValue Op0 = N->getOperand(0);
SDValue Op1 = N->getOperand(1);

// If the input also needs to be promoted, do that first so we can get a
// get a good idea for the output type.
if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
    == TargetLowering::TypePromoteInteger) {
  SDValue In = GetPromotedInteger(Op0);

  // If the new type is larger than NVT, use it. We probably won't need to
  // promote it again.
  EVT SVT = In.getValueType().getScalarType();
  if (SVT.bitsGE(NVT)) {
    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
    return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
  }
}

return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
648}

650SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
unsigned NewOpc = N->getOpcode();
SDLoc dl(N);

// If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
// not Legal, check to see if we can use FP_TO_SINT instead.  (If both UINT
// and SINT conversions are Custom, there is no way to tell which is
// preferable. We choose SINT because that's the right thing on PPC.)
if (N->getOpcode() == ISD::FP_TO_UINT &&
    !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
    TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
  NewOpc = ISD::FP_TO_SINT;

if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
    !TLI.isOperationLegal(ISD::STRICT_FP_TO_UINT, NVT) &&
    TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
  NewOpc = ISD::STRICT_FP_TO_SINT;

SDValue Res;
if (N->isStrictFPOpcode()) {
  Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
                    {N->getOperand(0), N->getOperand(1)});
  // Legalize the chain result - switch anything that used the old chain to
  // use the new one.
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
} else
  Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));

// Assert that the converted value fits in the original type.  If it doesn't
// (eg: because the value being converted is too big), then the result of the
// original operation was undefined anyway, so the assert is still correct.
//
// NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
//   before legalization: fp-to-uint16, 65534. -> 0xfffe
//   after legalization: fp-to-sint32, 65534. -> 0x0000fffe
return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
                    N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
                   ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
                   DAG.getValueType(N->getValueType(0).getScalarType()));
690}

692SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
// Promote the result type, while keeping the original width in Op1.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
                   N->getOperand(1));
698}

700SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);

return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
705}

707SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);

SDValue Res =
    DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));

// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
718}

720SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);

if (getTypeAction(N->getOperand(0).getValueType())
    == TargetLowering::TypePromoteInteger) {
  SDValue Res = GetPromotedInteger(N->getOperand(0));
  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!")(static_cast <bool> (Res.getValueType().bitsLE(NVT) &&
 "Extension doesn't make sense!") ? void (0) : __assert_fail (
"Res.getValueType().bitsLE(NVT) && \"Extension doesn't make sense!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 727
, __extension__ __PRETTY_FUNCTION__));

  // If the result and operand types are the same after promotion, simplify
  // to an in-register extension.
  if (NVT == Res.getValueType()) {
    // The high bits are not guaranteed to be anything.  Insert an extend.
    if (N->getOpcode() == ISD::SIGN_EXTEND)
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
                         DAG.getValueType(N->getOperand(0).getValueType()));
    if (N->getOpcode() == ISD::ZERO_EXTEND)
      return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
    assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!")(static_cast <bool> (N->getOpcode() == ISD::ANY_EXTEND
 && "Unknown integer extension!") ? void (0) : __assert_fail
 ("N->getOpcode() == ISD::ANY_EXTEND && \"Unknown integer extension!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 738
, __extension__ __PRETTY_FUNCTION__));
    return Res;
  }
}

// Otherwise, just extend the original operand all the way to the larger type.
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
745}

747SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDLoad(N) &&
 "Indexed load during type legalization!") ? void (0) : __assert_fail
 ("ISD::isUNINDEXEDLoad(N) && \"Indexed load during type legalization!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 748
, __extension__ __PRETTY_FUNCTION__));
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
ISD::LoadExtType ExtType =
  ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
SDLoc dl(N);
SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
                             N->getMemoryVT(), N->getMemOperand());

// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
760}

762SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());

ISD::LoadExtType ExtType = N->getExtensionType();
if (ExtType == ISD::NON_EXTLOAD)
  ExtType = ISD::EXTLOAD;

SDLoc dl(N);
SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
                                N->getOffset(), N->getMask(), ExtPassThru,
                                N->getMemoryVT(), N->getMemOperand(),
                                N->getAddressingMode(), ExtType,
                                N->isExpandingLoad());
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
780}

782SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
assert(NVT == ExtPassThru.getValueType() &&(static_cast <bool> (NVT == ExtPassThru.getValueType() &&
 "Gather result type and the passThru argument type should be the same"
) ? void (0) : __assert_fail ("NVT == ExtPassThru.getValueType() && \"Gather result type and the passThru argument type should be the same\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 786
, __extension__ __PRETTY_FUNCTION__))
    "Gather result type and the passThru argument type should be the same")(static_cast <bool> (NVT == ExtPassThru.getValueType() &&
 "Gather result type and the passThru argument type should be the same"
) ? void (0) : __assert_fail ("NVT == ExtPassThru.getValueType() && \"Gather result type and the passThru argument type should be the same\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 786
, __extension__ __PRETTY_FUNCTION__));

ISD::LoadExtType ExtType = N->getExtensionType();
if (ExtType == ISD::NON_EXTLOAD)
  ExtType = ISD::EXTLOAD;

SDLoc dl(N);
SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
                 N->getIndex(), N->getScale() };
SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
                                  N->getMemoryVT(), dl, Ops,
                                  N->getMemOperand(), N->getIndexType(),
                                  ExtType);
// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
return Res;
803}

805/// Promote the overflow flag of an overflowing arithmetic node.
806SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
// Change the return type of the boolean result while obeying
// getSetCCResultType.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
EVT VT = N->getValueType(0);
EVT SVT = getSetCCResultType(VT);
SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
unsigned NumOps = N->getNumOperands();
assert(NumOps <= 3 && "Too many operands")(static_cast <bool> (NumOps <= 3 && "Too many operands"
) ? void (0) : __assert_fail ("NumOps <= 3 && \"Too many operands\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 814
, __extension__ __PRETTY_FUNCTION__));
if (NumOps == 3)
  Ops[2] = N->getOperand(2);

SDLoc dl(N);
SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
                          makeArrayRef(Ops, NumOps));

// Modified the sum result - switch anything that used the old sum to use
// the new one.
ReplaceValueWith(SDValue(N, 0), Res);

// Convert to the expected type.
return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
828}

830SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
// If the promoted type is legal, we can convert this to:
//   1. ANY_EXTEND iN to iM
//   2. SHL by M-N
//   3. [US][ADD|SUB|SHL]SAT
//   4. L/ASHR by M-N
// Else it is more efficient to convert this to a min and a max
// operation in the higher precision arithmetic.
SDLoc dl(N);
SDValue Op1 = N->getOperand(0);
SDValue Op2 = N->getOperand(1);
unsigned OldBits = Op1.getScalarValueSizeInBits();

unsigned Opcode = N->getOpcode();
bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;

SDValue Op1Promoted, Op2Promoted;
if (IsShift) {
  Op1Promoted = GetPromotedInteger(Op1);
  Op2Promoted = ZExtPromotedInteger(Op2);
} else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
  Op1Promoted = ZExtPromotedInteger(Op1);
  Op2Promoted = ZExtPromotedInteger(Op2);
} else {
  Op1Promoted = SExtPromotedInteger(Op1);
  Op2Promoted = SExtPromotedInteger(Op2);
}
EVT PromotedType = Op1Promoted.getValueType();
unsigned NewBits = PromotedType.getScalarSizeInBits();

if (Opcode == ISD::UADDSAT) {
  APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
  SDValue Add =
      DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
  return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
}

// USUBSAT can always be promoted as long as we have zero-extended the args.
if (Opcode == ISD::USUBSAT)
  return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
                     Op2Promoted);

// Shift cannot use a min/max expansion, we can't detect overflow if all of
// the bits have been shifted out.
if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
  unsigned ShiftOp;
  switch (Opcode) {
  case ISD::SADDSAT:
  case ISD::SSUBSAT:
  case ISD::SSHLSAT:
    ShiftOp = ISD::SRA;
    break;
  case ISD::USHLSAT:
    ShiftOp = ISD::SRL;
    break;
  default:
    llvm_unreachable("Expected opcode to be signed or unsigned saturation "::llvm::llvm_unreachable_internal("Expected opcode to be signed or unsigned saturation "
 "addition, subtraction or left shift", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 888)
                     "addition, subtraction or left shift")::llvm::llvm_unreachable_internal("Expected opcode to be signed or unsigned saturation "
 "addition, subtraction or left shift", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 888);
  }

  unsigned SHLAmount = NewBits - OldBits;
  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
  Op1Promoted =
      DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
  if (!IsShift)
    Op2Promoted =
        DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);

  SDValue Result =
      DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
}

unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
SDValue Result =
    DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
return Result;
915}

917SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
// Can just promote the operands then continue with operation.
SDLoc dl(N);
SDValue Op1Promoted, Op2Promoted;
bool Signed =
    N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
bool Saturating =
    N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
if (Signed) {
  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
} else {
  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
}
EVT OldType = N->getOperand(0).getValueType();
EVT PromotedType = Op1Promoted.getValueType();
unsigned DiffSize =
    PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();

if (Saturating) {
  // Promoting the operand and result values changes the saturation width,
  // which is extends the values that we clamp to on saturation. This could be
  // resolved by shifting one of the operands the same amount, which would
  // also shift the result we compare against, then shifting back.
  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
                            DAG.getConstant(DiffSize, dl, ShiftTy));
  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
                               Op2Promoted, N->getOperand(2));
  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
                     DAG.getConstant(DiffSize, dl, ShiftTy));
}
return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
                   N->getOperand(2));
953}

955static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl,
                                   unsigned SatW, bool Signed,
                                   const TargetLowering &TLI,
                                   SelectionDAG &DAG) {
EVT VT = V.getValueType();
unsigned VTW = VT.getScalarSizeInBits();

if (!Signed) {
  // Saturate to the unsigned maximum by getting the minimum of V and the
  // maximum.
  return DAG.getNode(ISD::UMIN, dl, VT, V,
                     DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
                                     dl, VT));
}

// Saturate to the signed maximum (the low SatW - 1 bits) by taking the
// signed minimum of it and V.
V = DAG.getNode(ISD::SMIN, dl, VT, V,
                DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
                                dl, VT));
// Saturate to the signed minimum (the high SatW + 1 bits) by taking the
// signed maximum of it and V.
V = DAG.getNode(ISD::SMAX, dl, VT, V,
                DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
                                dl, VT));
return V;
981}

983static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS,
                               unsigned Scale, const TargetLowering &TLI,
                               SelectionDAG &DAG, unsigned SatW = 0) {
EVT VT = LHS.getValueType();
unsigned VTSize = VT.getScalarSizeInBits();
bool Signed = N->getOpcode() == ISD::SDIVFIX ||
              N->getOpcode() == ISD::SDIVFIXSAT;
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
                  N->getOpcode() == ISD::UDIVFIXSAT;

SDLoc dl(N);
// Widen the types by a factor of two. This is guaranteed to expand, since it
// will always have enough high bits in the LHS to shift into.
EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
if (VT.isVector())
  WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
                            VT.getVectorElementCount());
if (Signed) {
  LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
  RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
} else {
  LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
  RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
}

SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
                                      DAG);
assert(Res && "Expanding DIVFIX with wide type failed?")(static_cast <bool> (Res && "Expanding DIVFIX with wide type failed?"
) ? void (0) : __assert_fail ("Res && \"Expanding DIVFIX with wide type failed?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1010
, __extension__ __PRETTY_FUNCTION__));
if (Saturating) {
  // If the caller has told us to saturate at something less, use that width
  // instead of the type before doubling. However, it cannot be more than
  // what we just widened!
  assert(SatW <= VTSize &&(static_cast <bool> (SatW <= VTSize && "Tried to saturate to more than the original type?"
) ? void (0) : __assert_fail ("SatW <= VTSize && \"Tried to saturate to more than the original type?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1016
, __extension__ __PRETTY_FUNCTION__))
         "Tried to saturate to more than the original type?")(static_cast <bool> (SatW <= VTSize && "Tried to saturate to more than the original type?"
) ? void (0) : __assert_fail ("SatW <= VTSize && \"Tried to saturate to more than the original type?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1016
, __extension__ __PRETTY_FUNCTION__));
  Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
                              TLI, DAG);
}
return DAG.getZExtOrTrunc(Res, dl, VT);
1021}

1023SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
SDLoc dl(N);
SDValue Op1Promoted, Op2Promoted;
bool Signed = N->getOpcode() == ISD::SDIVFIX ||
              N->getOpcode() == ISD::SDIVFIXSAT;
bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
                  N->getOpcode() == ISD::UDIVFIXSAT;
if (Signed) {
  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
} else {
  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
}
EVT PromotedType = Op1Promoted.getValueType();
unsigned Scale = N->getConstantOperandVal(2);

// If the type is already legal and the operation is legal in that type, we
// should not early expand.
if (TLI.isTypeLegal(PromotedType)) {
  TargetLowering::LegalizeAction Action =
      TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
  if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
    EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
    unsigned Diff = PromotedType.getScalarSizeInBits() -
                    N->getValueType(0).getScalarSizeInBits();
    if (Saturating)
      Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
                                DAG.getConstant(Diff, dl, ShiftTy));
    SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
                              Op2Promoted, N->getOperand(2));
    if (Saturating)
      Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
                        DAG.getConstant(Diff, dl, ShiftTy));
    return Res;
  }
}

// See if we can perform the division in this type without expanding.
if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
                                      Op2Promoted, Scale, DAG)) {
  if (Saturating)
    Res = SaturateWidenedDIVFIX(Res, dl,
                                N->getValueType(0).getScalarSizeInBits(),
                                Signed, TLI, DAG);
  return Res;
}
// If we cannot, expand it to twice the type width. If we are saturating, give
// it the original width as a saturating width so we don't need to emit
// two saturations.
return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
                          N->getValueType(0).getScalarSizeInBits());
1075}

1077SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
if (ResNo == 1)
  return PromoteIntRes_Overflow(N);

// The operation overflowed iff the result in the larger type is not the
// sign extension of its truncation to the original type.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtPromotedInteger(N->getOperand(1));
EVT OVT = N->getOperand(0).getValueType();
EVT NVT = LHS.getValueType();
SDLoc dl(N);

// Do the arithmetic in the larger type.
unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);

// Calculate the overflow flag: sign extend the arithmetic result from
// the original type.
SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
                          DAG.getValueType(OVT));
// Overflowed if and only if this is not equal to Res.
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);

// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(N, 1), Ofl);

return Res;
1104}

1106SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
SDValue Mask = N->getOperand(0);

SDValue LHS = GetPromotedInteger(N->getOperand(1));
SDValue RHS = GetPromotedInteger(N->getOperand(2));

unsigned Opcode = N->getOpcode();
return Opcode == ISD::VP_SELECT
           ? DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
                         N->getOperand(3))
           : DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS,
                         RHS);
1118}

1120SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
SDValue LHS = GetPromotedInteger(N->getOperand(2));
SDValue RHS = GetPromotedInteger(N->getOperand(3));
return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
                   LHS.getValueType(), N->getOperand(0),
                   N->getOperand(1), LHS, RHS, N->getOperand(4));
1126}

1128SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
EVT InVT = N->getOperand(OpNo).getValueType();
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));

EVT SVT = getSetCCResultType(InVT);

// If we got back a type that needs to be promoted, this likely means the
// the input type also needs to be promoted. So get the promoted type for
// the input and try the query again.
if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
    InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
    SVT = getSetCCResultType(InVT);
  } else {
    // Input type isn't promoted, just use the default promoted type.
    SVT = NVT;
  }
}

SDLoc dl(N);
assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&(static_cast <bool> (SVT.isVector() == N->getOperand
(OpNo).getValueType().isVector() && "Vector compare must return a vector result!"
) ? void (0) : __assert_fail ("SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && \"Vector compare must return a vector result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1150
, __extension__ __PRETTY_FUNCTION__))
       "Vector compare must return a vector result!")(static_cast <bool> (SVT.isVector() == N->getOperand
(OpNo).getValueType().isVector() && "Vector compare must return a vector result!"
) ? void (0) : __assert_fail ("SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() && \"Vector compare must return a vector result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1150
, __extension__ __PRETTY_FUNCTION__));

// Get the SETCC result using the canonical SETCC type.
SDValue SetCC;
if (N->isStrictFPOpcode()) {
  EVT VTs[] = {SVT, MVT::Other};
  SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
                     N->getOperand(2), N->getOperand(3)};
  SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
  // Legalize the chain result - switch anything that used the old chain to
  // use the new one.
  ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
} else
  SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
                      N->getOperand(1), N->getOperand(2));

// Convert to the expected type.
return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1168}

1170SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
SDValue LHS = GetPromotedInteger(N->getOperand(0));
SDValue RHS = N->getOperand(1);
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
  RHS = ZExtPromotedInteger(RHS);
if (N->getOpcode() != ISD::VP_SHL)
  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                   N->getOperand(2), N->getOperand(3));
1179}

1181SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
                   Op.getValueType(), Op, N->getOperand(1));
1185}

1187SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
// The input may have strange things in the top bits of the registers, but
// these operations don't care.  They may have weird bits going out, but
// that too is okay if they are integer operations.
SDValue LHS = GetPromotedInteger(N->getOperand(0));
SDValue RHS = GetPromotedInteger(N->getOperand(1));
if (N->getNumOperands() == 2)
  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
assert(N->getNumOperands() == 4 && "Unexpected number of operands!")(static_cast <bool> (N->getNumOperands() == 4 &&
 "Unexpected number of operands!") ? void (0) : __assert_fail
 ("N->getNumOperands() == 4 && \"Unexpected number of operands!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1195
, __extension__ __PRETTY_FUNCTION__));
assert(N->isVPOpcode() && "Expected VP opcode")(static_cast <bool> (N->isVPOpcode() && "Expected VP opcode"
) ? void (0) : __assert_fail ("N->isVPOpcode() && \"Expected VP opcode\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1196
, __extension__ __PRETTY_FUNCTION__));
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                   N->getOperand(2), N->getOperand(3));
1199}

1201SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
// Sign extend the input.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtPromotedInteger(N->getOperand(1));
if (N->getNumOperands() == 2)
  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
assert(N->getNumOperands() == 4 && "Unexpected number of operands!")(static_cast <bool> (N->getNumOperands() == 4 &&
 "Unexpected number of operands!") ? void (0) : __assert_fail
 ("N->getNumOperands() == 4 && \"Unexpected number of operands!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1207
, __extension__ __PRETTY_FUNCTION__));
assert(N->isVPOpcode() && "Expected VP opcode")(static_cast <bool> (N->isVPOpcode() && "Expected VP opcode"
) ? void (0) : __assert_fail ("N->isVPOpcode() && \"Expected VP opcode\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1208
, __extension__ __PRETTY_FUNCTION__));
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                   N->getOperand(2), N->getOperand(3));
1211}

1213SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
// Zero extend the input.
SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
if (N->getNumOperands() == 2)
  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
assert(N->getNumOperands() == 4 && "Unexpected number of operands!")(static_cast <bool> (N->getNumOperands() == 4 &&
 "Unexpected number of operands!") ? void (0) : __assert_fail
 ("N->getNumOperands() == 4 && \"Unexpected number of operands!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1219
, __extension__ __PRETTY_FUNCTION__));
assert(N->isVPOpcode() && "Expected VP opcode")(static_cast <bool> (N->isVPOpcode() && "Expected VP opcode"
) ? void (0) : __assert_fail ("N->isVPOpcode() && \"Expected VP opcode\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1220
, __extension__ __PRETTY_FUNCTION__));
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                   N->getOperand(2), N->getOperand(3));
1223}

1225SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
// It doesn't matter if we sign extend or zero extend in the inputs. So do
// whatever is best for the target.
SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
return DAG.getNode(N->getOpcode(), SDLoc(N),
                   LHS.getValueType(), LHS, RHS);
1232}

1234SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
// The input value must be properly sign extended.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = N->getOperand(1);
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
  RHS = ZExtPromotedInteger(RHS);
if (N->getOpcode() != ISD::VP_ASHR)
  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                   N->getOperand(2), N->getOperand(3));
1244}

1246SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
// The input value must be properly zero extended.
SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
SDValue RHS = N->getOperand(1);
if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
  RHS = ZExtPromotedInteger(RHS);
if (N->getOpcode() != ISD::VP_LSHR)
  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
                   N->getOperand(2), N->getOperand(3));
1256}

1258SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
// Lower the rotate to shifts and ORs which can be promoted.
SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
ReplaceValueWith(SDValue(N, 0), Res);
return SDValue();
1263}

1265SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
SDValue Hi = GetPromotedInteger(N->getOperand(0));
SDValue Lo = GetPromotedInteger(N->getOperand(1));
SDValue Amt = GetPromotedInteger(N->getOperand(2));

SDLoc DL(N);
EVT OldVT = N->getOperand(0).getValueType();
EVT VT = Lo.getValueType();
unsigned Opcode = N->getOpcode();
bool IsFSHR = Opcode == ISD::FSHR;
unsigned OldBits = OldVT.getScalarSizeInBits();
unsigned NewBits = VT.getScalarSizeInBits();

// Amount has to be interpreted modulo the old bit width.
Amt = DAG.getNode(ISD::UREM, DL, VT, Amt, DAG.getConstant(OldBits, DL, VT));

// If the promoted type is twice the size (or more), then we use the
// traditional funnel 'double' shift codegen. This isn't necessary if the
// shift amount is constant.
// fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
// fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
    !TLI.isOperationLegalOrCustom(Opcode, VT)) {
  SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
  Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
  Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
  SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
  Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
  if (!IsFSHR)
    Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
  return Res;
}

// Shift Lo up to occupy the upper bits of the promoted type.
SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT);
Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);

// Increase Amount to shift the result into the lower bits of the promoted
// type.
if (IsFSHR)
  Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, ShiftOffset);

return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1308}

1310SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res;
SDValue InOp = N->getOperand(0);
SDLoc dl(N);

switch (getTypeAction(InOp.getValueType())) {
default: llvm_unreachable("Unknown type action!")::llvm::llvm_unreachable_internal("Unknown type action!", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 1317);
case TargetLowering::TypeLegal:
case TargetLowering::TypeExpandInteger:
  Res = InOp;
  break;
case TargetLowering::TypePromoteInteger:
  Res = GetPromotedInteger(InOp);
  break;
case TargetLowering::TypeSplitVector: {
  EVT InVT = InOp.getValueType();
  assert(InVT.isVector() && "Cannot split scalar types")(static_cast <bool> (InVT.isVector() && "Cannot split scalar types"
) ? void (0) : __assert_fail ("InVT.isVector() && \"Cannot split scalar types\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1327
, __extension__ __PRETTY_FUNCTION__));
  ElementCount NumElts = InVT.getVectorElementCount();
  assert(NumElts == NVT.getVectorElementCount() &&(static_cast <bool> (NumElts == NVT.getVectorElementCount
() && "Dst and Src must have the same number of elements"
) ? void (0) : __assert_fail ("NumElts == NVT.getVectorElementCount() && \"Dst and Src must have the same number of elements\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1330
, __extension__ __PRETTY_FUNCTION__))
         "Dst and Src must have the same number of elements")(static_cast <bool> (NumElts == NVT.getVectorElementCount
() && "Dst and Src must have the same number of elements"
) ? void (0) : __assert_fail ("NumElts == NVT.getVectorElementCount() && \"Dst and Src must have the same number of elements\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1330
, __extension__ __PRETTY_FUNCTION__));
  assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&(static_cast <bool> (isPowerOf2_32(NumElts.getKnownMinValue
()) && "Promoted vector type must be a power of two")
 ? void (0) : __assert_fail ("isPowerOf2_32(NumElts.getKnownMinValue()) && \"Promoted vector type must be a power of two\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1332
, __extension__ __PRETTY_FUNCTION__))
         "Promoted vector type must be a power of two")(static_cast <bool> (isPowerOf2_32(NumElts.getKnownMinValue
()) && "Promoted vector type must be a power of two")
 ? void (0) : __assert_fail ("isPowerOf2_32(NumElts.getKnownMinValue()) && \"Promoted vector type must be a power of two\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1332
, __extension__ __PRETTY_FUNCTION__));

  SDValue EOp1, EOp2;
  GetSplitVector(InOp, EOp1, EOp2);

  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
                                 NumElts.divideCoefficientBy(2));
  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);

  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
}
case TargetLowering::TypeWidenVector: {
  SDValue WideInOp = GetWidenedVector(InOp);

  // Truncate widened InOp.
  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
                                 N->getValueType(0).getScalarType(), NumElem);
  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);

  // Zero extend so that the elements are of same type as those of NVT
  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
                               NumElem);
  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);

  // Extract the low NVT subvector.
  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
}
}

// Truncate to NVT instead of VT
return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1366}

1368SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
if (ResNo == 1)
  return PromoteIntRes_Overflow(N);

// The operation overflowed iff the result in the larger type is not the
// zero extension of its truncation to the original type.
SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
EVT OVT = N->getOperand(0).getValueType();
EVT NVT = LHS.getValueType();
SDLoc dl(N);

// Do the arithmetic in the larger type.
unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);

// Calculate the overflow flag: zero extend the arithmetic result from
// the original type.
SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
// Overflowed if and only if this is not equal to Res.
Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);

// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(N, 1), Ofl);

return Res;
1394}

1396// Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1397// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1398// the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
1399SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
if (ResNo == 1)
  return PromoteIntRes_Overflow(N);

// We need to sign-extend the operands so the carry value computed by the
// wide operation will be equivalent to the carry value computed by the
// narrow operation.
// An ADDCARRY can generate carry only if any of the operands has its
// most significant bit set. Sign extension propagates the most significant
// bit into the higher bits which means the extra bit that the narrow
// addition would need (i.e. the carry) will be propagated through the higher
// bits of the wide addition.
// A SUBCARRY can generate borrow only if LHS < RHS and this property will be
// preserved by sign extension.
SDValue LHS = SExtPromotedInteger(N->getOperand(0));
SDValue RHS = SExtPromotedInteger(N->getOperand(1));

EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};

// Do the arithmetic in the wide type.
SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
                          LHS, RHS, N->getOperand(2));

// Update the users of the original carry/borrow value.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));

return SDValue(Res.getNode(), 0);
1426}

1428SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
                                                     unsigned ResNo) {
assert(ResNo == 1 && "Don't know how to promote other results yet.")(static_cast <bool> (ResNo == 1 && "Don't know how to promote other results yet."
) ? void (0) : __assert_fail ("ResNo == 1 && \"Don't know how to promote other results yet.\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1430
, __extension__ __PRETTY_FUNCTION__));
return PromoteIntRes_Overflow(N);
1432}

1434SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
EVT OVT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);

// If a larger ABS or SMAX isn't supported by the target, try to expand now.
// If we expand later we'll end up sign extending more than just the sra input
// in sra+xor+sub expansion.
if (!OVT.isVector() &&
    !TLI.isOperationLegalOrCustomOrPromote(ISD::ABS, NVT) &&
    !TLI.isOperationLegal(ISD::SMAX, NVT)) {
  if (SDValue Res = TLI.expandABS(N, DAG))
    return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
}

SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1450}

1452SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
// Promote the overflow bit trivially.
if (ResNo == 1)
  return PromoteIntRes_Overflow(N);

SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
SDLoc DL(N);
EVT SmallVT = LHS.getValueType();

// To determine if the result overflowed in a larger type, we extend the
// input to the larger type, do the multiply (checking if it overflows),
// then also check the high bits of the result to see if overflow happened
// there.
if (N->getOpcode() == ISD::SMULO) {
  LHS = SExtPromotedInteger(LHS);
  RHS = SExtPromotedInteger(RHS);
} else {
  LHS = ZExtPromotedInteger(LHS);
  RHS = ZExtPromotedInteger(RHS);
}
SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);

// Overflow occurred if it occurred in the larger type, or if the high part
// of the result does not zero/sign-extend the low part.  Check this second
// possibility first.
SDValue Overflow;
if (N->getOpcode() == ISD::UMULO) {
  // Unsigned overflow occurred if the high part is non-zero.
  unsigned Shift = SmallVT.getScalarSizeInBits();
  EVT ShiftTy = TLI.getShiftAmountTy(Mul.getValueType(), DAG.getDataLayout());
  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
                           DAG.getConstant(Shift, DL, ShiftTy));
  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
                          DAG.getConstant(0, DL, Hi.getValueType()),
                          ISD::SETNE);
} else {
  // Signed overflow occurred if the high part does not sign extend the low.
  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
                             Mul, DAG.getValueType(SmallVT));
  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
}

// The only other way for overflow to occur is if the multiplication in the
// larger type itself overflowed.
Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
                       SDValue(Mul.getNode(), 1));

// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(N, 1), Overflow);
return Mul;
1503}

1505SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
                                             N->getValueType(0)));
1508}

1510SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));

APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1515}

1517SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
SDValue Chain = N->getOperand(0); // Get the chain.
SDValue Ptr = N->getOperand(1); // Get the pointer.
EVT VT = N->getValueType(0);
SDLoc dl(N);

MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
// The argument is passed as NumRegs registers of type RegVT.

SmallVector<SDValue, 8> Parts(NumRegs);
for (unsigned i = 0; i < NumRegs; ++i) {
  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
                          N->getConstantOperandVal(3));
  Chain = Parts[i].getValue(1);
}

// Handle endianness of the load.
if (DAG.getDataLayout().isBigEndian())
  std::reverse(Parts.begin(), Parts.end());

// Assemble the parts in the promoted type.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
for (unsigned i = 1; i < NumRegs; ++i) {
  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
  // Shift it to the right position and "or" it in.
  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
                     DAG.getConstant(i * RegVT.getSizeInBits(), dl,
                                     TLI.getPointerTy(DAG.getDataLayout())));
  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
}

// Modified the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Chain);

return Res;
1555}

1557//===----------------------------------------------------------------------===//
1558//  Integer Operand Promotion
1559//===----------------------------------------------------------------------===//

1561/// PromoteIntegerOperand - This method is called when the specified operand of
1562/// the specified node is found to need promotion.  At this point, all of the
1563/// result types of the node are known to be legal, but other operands of the
1564/// node may need promotion or expansion as well as the specified one.
1565bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
           dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Promote integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
);
SDValue Res = SDValue();
if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Node has been custom lowered, done\n"
; } } while (false);
  return false;
}

switch (N->getOpcode()) {
  default:
#ifndef NDEBUG
  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
  N->dump(&DAG); dbgs() << "\n";
#endif
  llvm_unreachable("Do not know how to promote this operator's operand!")::llvm::llvm_unreachable_internal("Do not know how to promote this operator's operand!"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1580
);

case ISD::ANY_EXTEND:   Res = PromoteIntOp_ANY_EXTEND(N); break;
case ISD::ATOMIC_STORE:
  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
  break;
case ISD::BITCAST:      Res = PromoteIntOp_BITCAST(N); break;
case ISD::BR_CC:        Res = PromoteIntOp_BR_CC(N, OpNo); break;
case ISD::BRCOND:       Res = PromoteIntOp_BRCOND(N, OpNo); break;
case ISD::BUILD_PAIR:   Res = PromoteIntOp_BUILD_PAIR(N); break;
case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
case ISD::INSERT_VECTOR_ELT:
                        Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
case ISD::SCALAR_TO_VECTOR:
                        Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
case ISD::SPLAT_VECTOR:
                        Res = PromoteIntOp_SPLAT_VECTOR(N); break;
case ISD::VSELECT:
case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
case ISD::SIGN_EXTEND:  Res = PromoteIntOp_SIGN_EXTEND(N); break;
case ISD::SINT_TO_FP:   Res = PromoteIntOp_SINT_TO_FP(N); break;
case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
                                                 OpNo); break;
case ISD::MSTORE:       Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
                                                  OpNo); break;
case ISD::MLOAD:        Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
                                                  OpNo); break;
case ISD::MGATHER:  Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
                                               OpNo); break;
case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
                                                OpNo); break;
case ISD::TRUNCATE:     Res = PromoteIntOp_TRUNCATE(N); break;
case ISD::FP16_TO_FP:
case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
case ISD::STRICT_UINT_TO_FP:  Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;

case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
case ISD::ROTL:
case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;

case ISD::SADDO_CARRY:
case ISD::SSUBO_CARRY:
case ISD::ADDCARRY:
case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;

case ISD::FRAMEADDR:
case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;

case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;

case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
case ISD::UMULFIXSAT:
case ISD::SDIVFIX:
case ISD::SDIVFIXSAT:
case ISD::UDIVFIX:
case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;

case ISD::FPOWI:
case ISD::STRICT_FPOWI: Res = PromoteIntOp_FPOWI(N); break;

case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
case ISD::VP_REDUCE_ADD:
case ISD::VP_REDUCE_MUL:
case ISD::VP_REDUCE_AND:
case ISD::VP_REDUCE_OR:
case ISD::VP_REDUCE_XOR:
case ISD::VP_REDUCE_SMAX:
case ISD::VP_REDUCE_SMIN:
case ISD::VP_REDUCE_UMAX:
case ISD::VP_REDUCE_UMIN:
  Res = PromoteIntOp_VP_REDUCE(N, OpNo);
  break;

case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
}

// If the result is null, the sub-method took care of registering results etc.
if (!Res.getNode()) return false;

// If the result is N, the sub-method updated N in place.  Tell the legalizer
// core about this.
if (Res.getNode() == N)
  return true;

const bool IsStrictFp = N->isStrictFPOpcode();
assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) &&
 "Invalid operand expansion") ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) && \"Invalid operand expansion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1687
, __extension__ __PRETTY_FUNCTION__))
       N->getNumValues() == (IsStrictFp ? 2 : 1) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) &&
 "Invalid operand expansion") ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) && \"Invalid operand expansion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1687
, __extension__ __PRETTY_FUNCTION__))
       "Invalid operand expansion")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) &&
 "Invalid operand expansion") ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == (IsStrictFp ? 2 : 1) && \"Invalid operand expansion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1687
, __extension__ __PRETTY_FUNCTION__));
LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << "     with: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Replacing: "; N->dump
(&DAG); dbgs() << "     with: "; Res.dump(); } } while
 (false)
           Res.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Replacing: "; N->dump
(&DAG); dbgs() << "     with: "; Res.dump(); } } while
 (false);

ReplaceValueWith(SDValue(N, 0), Res);
if (IsStrictFp)
  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));

return false;
1696}

1698/// PromoteSetCCOperands - Promote the operands of a comparison.  This code is
1699/// shared among BR_CC, SELECT_CC, and SETCC handlers.
1700void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
                                          ISD::CondCode CCCode) {
// We have to insert explicit sign or zero extends. Note that we could
// insert sign extends for ALL conditions. For those operations where either
// zero or sign extension would be valid, we ask the target which extension
// it would prefer.

// Signed comparisons always require sign extension.
if (ISD::isSignedIntSetCC(CCCode)) {
  LHS = SExtPromotedInteger(LHS);
  RHS = SExtPromotedInteger(RHS);
  return;
}

assert((ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) &&(static_cast <bool> ((ISD::isUnsignedIntSetCC(CCCode) ||
 ISD::isIntEqualitySetCC(CCCode)) && "Unknown integer comparison!"
) ? void (0) : __assert_fail ("(ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && \"Unknown integer comparison!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1715
, __extension__ __PRETTY_FUNCTION__))
       "Unknown integer comparison!")(static_cast <bool> ((ISD::isUnsignedIntSetCC(CCCode) ||
 ISD::isIntEqualitySetCC(CCCode)) && "Unknown integer comparison!"
) ? void (0) : __assert_fail ("(ISD::isUnsignedIntSetCC(CCCode) || ISD::isIntEqualitySetCC(CCCode)) && \"Unknown integer comparison!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1715
, __extension__ __PRETTY_FUNCTION__));

SDValue OpL = GetPromotedInteger(LHS);
SDValue OpR = GetPromotedInteger(RHS);

if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
  // The target would prefer to promote the comparison operand with sign
  // extension. Honor that unless the promoted values are already zero
  // extended.
  unsigned OpLEffectiveBits =
      DAG.computeKnownBits(OpL).countMaxActiveBits();
  unsigned OpREffectiveBits =
      DAG.computeKnownBits(OpR).countMaxActiveBits();
  if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
      OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
    LHS = OpL;
    RHS = OpR;
    return;
  }

  // The promoted values aren't zero extended, use a sext_inreg.
  LHS = SExtPromotedInteger(LHS);
  RHS = SExtPromotedInteger(RHS);
  return;
}

// Prefer to promote the comparison operand with zero extension.

// If the width of OpL/OpR excluding the duplicated sign bits is no greater
// than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
// that we might not be able to remove.
unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
    OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
  LHS = OpL;
  RHS = OpR;
  return;
}

// Otherwise, use zext_inreg.
LHS = ZExtPromotedInteger(LHS);
RHS = ZExtPromotedInteger(RHS);
1758}

1760SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1763}

1765SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
                     N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1769}

1771SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
// This should only occur in unusual situations like bitcasting to an
// x86_fp80, so just turn it into a store+load
return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1775}

1777SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
assert(OpNo == 2 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 2 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 2 && \"Don't know how to promote this operand!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1778
, __extension__ __PRETTY_FUNCTION__));

SDValue LHS = N->getOperand(2);
SDValue RHS = N->getOperand(3);
PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());

// The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
// legal types.
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                              N->getOperand(1), LHS, RHS, N->getOperand(4)),
               0);
1789}

1791SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
assert(OpNo == 1 && "only know how to promote condition")(static_cast <bool> (OpNo == 1 && "only know how to promote condition"
) ? void (0) : __assert_fail ("OpNo == 1 && \"only know how to promote condition\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1792
, __extension__ __PRETTY_FUNCTION__));

// Promote all the way up to the canonical SetCC type.
SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);

// The chain (Op#0) and basic block destination (Op#2) are always legal types.
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
                                      N->getOperand(2)), 0);
1800}

1802SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
// Since the result type is legal, the operands must promote to it.
EVT OVT = N->getOperand(0).getValueType();
SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
SDValue Hi = GetPromotedInteger(N->getOperand(1));
assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?")(static_cast <bool> (Lo.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Lo.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1807
, __extension__ __PRETTY_FUNCTION__));
SDLoc dl(N);

Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
                 DAG.getConstant(OVT.getSizeInBits(), dl,
                                 TLI.getPointerTy(DAG.getDataLayout())));
return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1814}

1816SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
// The vector type is legal but the element type is not.  This implies
// that the vector is a power-of-two in length and that the element
// type does not have a strange size (eg: it is not i1).
EVT VecVT = N->getValueType(0);
unsigned NumElts = VecVT.getVectorNumElements();
assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&(static_cast <bool> (!((NumElts & 1) && (!TLI
.isTypeLegal(VecVT))) && "Legal vector of one illegal element?"
) ? void (0) : __assert_fail ("!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && \"Legal vector of one illegal element?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1823
, __extension__ __PRETTY_FUNCTION__))
       "Legal vector of one illegal element?")(static_cast <bool> (!((NumElts & 1) && (!TLI
.isTypeLegal(VecVT))) && "Legal vector of one illegal element?"
) ? void (0) : __assert_fail ("!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && \"Legal vector of one illegal element?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1823
, __extension__ __PRETTY_FUNCTION__));

// Promote the inserted value.  The type does not need to match the
// vector element type.  Check that any extra bits introduced will be
// truncated away.
assert(N->getOperand(0).getValueSizeInBits() >=(static_cast <bool> (N->getOperand(0).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
 "Type of inserted value narrower than vector element type!")
 ? void (0) : __assert_fail ("N->getOperand(0).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1830
, __extension__ __PRETTY_FUNCTION__))
       N->getValueType(0).getScalarSizeInBits() &&(static_cast <bool> (N->getOperand(0).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
 "Type of inserted value narrower than vector element type!")
 ? void (0) : __assert_fail ("N->getOperand(0).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1830
, __extension__ __PRETTY_FUNCTION__))
       "Type of inserted value narrower than vector element type!")(static_cast <bool> (N->getOperand(0).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
 "Type of inserted value narrower than vector element type!")
 ? void (0) : __assert_fail ("N->getOperand(0).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1830
, __extension__ __PRETTY_FUNCTION__));

SmallVector<SDValue, 16> NewOps;
for (unsigned i = 0; i < NumElts; ++i)
  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));

return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1837}

1839SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
                                                       unsigned OpNo) {
if (OpNo == 1) {
  // Promote the inserted value.  This is valid because the type does not
  // have to match the vector element type.

  // Check that any extra bits introduced will be truncated away.
  assert(N->getOperand(1).getValueSizeInBits() >=(static_cast <bool> (N->getOperand(1).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
 "Type of inserted value narrower than vector element type!")
 ? void (0) : __assert_fail ("N->getOperand(1).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1848
, __extension__ __PRETTY_FUNCTION__))
         N->getValueType(0).getScalarSizeInBits() &&(static_cast <bool> (N->getOperand(1).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
 "Type of inserted value narrower than vector element type!")
 ? void (0) : __assert_fail ("N->getOperand(1).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1848
, __extension__ __PRETTY_FUNCTION__))
         "Type of inserted value narrower than vector element type!")(static_cast <bool> (N->getOperand(1).getValueSizeInBits
() >= N->getValueType(0).getScalarSizeInBits() &&
 "Type of inserted value narrower than vector element type!")
 ? void (0) : __assert_fail ("N->getOperand(1).getValueSizeInBits() >= N->getValueType(0).getScalarSizeInBits() && \"Type of inserted value narrower than vector element type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1848
, __extension__ __PRETTY_FUNCTION__));
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                                GetPromotedInteger(N->getOperand(1)),
                                N->getOperand(2)),
                 0);
}

assert(OpNo == 2 && "Different operand and result vector types?")(static_cast <bool> (OpNo == 2 && "Different operand and result vector types?"
) ? void (0) : __assert_fail ("OpNo == 2 && \"Different operand and result vector types?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1855
, __extension__ __PRETTY_FUNCTION__));

// Promote the index.
SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
                                 TLI.getVectorIdxTy(DAG.getDataLayout()));
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                              N->getOperand(1), Idx), 0);
1862}

1864SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
// Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
// the operand in place.
return SDValue(DAG.UpdateNodeOperands(N,
                              GetPromotedInteger(N->getOperand(0))), 0);
1869}

1871SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
// Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
// operand in place.
return SDValue(
    DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1876}

1878SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Only know how to promote the condition!")(static_cast <bool> (OpNo == 0 && "Only know how to promote the condition!"
) ? void (0) : __assert_fail ("OpNo == 0 && \"Only know how to promote the condition!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1879
, __extension__ __PRETTY_FUNCTION__));
SDValue Cond = N->getOperand(0);
EVT OpTy = N->getOperand(1).getValueType();

if (N->getOpcode() == ISD::VSELECT)
  if (SDValue Res = WidenVSELECTMask(N))
    return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
                       Res, N->getOperand(1), N->getOperand(2));

// Promote all the way up to the canonical SetCC type.
EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
Cond = PromoteTargetBoolean(Cond, OpVT);

return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
                                      N->getOperand(2)), 0);
1894}

1896SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 0 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 0 && \"Don't know how to promote this operand!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1897
, __extension__ __PRETTY_FUNCTION__));

SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());

// The CC (#4) and the possible return values (#2 and #3) have legal types.
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
                              N->getOperand(3), N->getOperand(4)), 0);
1906}

1908SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 0 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 0 && \"Don't know how to promote this operand!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1909
, __extension__ __PRETTY_FUNCTION__));

SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());

// The CC (#2) is always legal.
return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1917}

1919SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                              ZExtPromotedInteger(N->getOperand(1))), 0);
1922}

1924SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
SDLoc dl(N);
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
                   Op, DAG.getValueType(N->getOperand(0).getValueType()));
1930}

1932SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N,
                              SExtPromotedInteger(N->getOperand(0))), 0);
1935}

1937SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                              SExtPromotedInteger(N->getOperand(1))), 0);
1940}

1942SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDStore(N) &&
 "Indexed store during type legalization!") ? void (0) : __assert_fail
 ("ISD::isUNINDEXEDStore(N) && \"Indexed store during type legalization!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1943
, __extension__ __PRETTY_FUNCTION__));
SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
SDLoc dl(N);

SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.

// Truncate the value and store the result.
return DAG.getTruncStore(Ch, dl, Val, Ptr,
                         N->getMemoryVT(), N->getMemOperand());
1952}

1954SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
                                            unsigned OpNo) {
SDValue DataOp = N->getValue();
SDValue Mask = N->getMask();

if (OpNo == 4) {
  // The Mask. Update in place.
  EVT DataVT = DataOp.getValueType();
  Mask = PromoteTargetBoolean(Mask, DataVT);
  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
  NewOps[4] = Mask;
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}

assert(OpNo == 1 && "Unexpected operand for promotion")(static_cast <bool> (OpNo == 1 && "Unexpected operand for promotion"
) ? void (0) : __assert_fail ("OpNo == 1 && \"Unexpected operand for promotion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1968
, __extension__ __PRETTY_FUNCTION__));
DataOp = GetPromotedInteger(DataOp);

return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
                          N->getOffset(), Mask, N->getMemoryVT(),
                          N->getMemOperand(), N->getAddressingMode(),
                          /*IsTruncating*/ true, N->isCompressingStore());
1975}

1977SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
                                           unsigned OpNo) {
assert(OpNo == 3 && "Only know how to promote the mask!")(static_cast <bool> (OpNo == 3 && "Only know how to promote the mask!"
) ? void (0) : __assert_fail ("OpNo == 3 && \"Only know how to promote the mask!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 1979
, __extension__ __PRETTY_FUNCTION__));
EVT DataVT = N->getValueType(0);
SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
NewOps[OpNo] = Mask;
SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
if (Res == N)
  return SDValue(Res, 0);

// Update triggered CSE, do our own replacement since caller can't.
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
return SDValue();
1992}

1994SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
                                             unsigned OpNo) {
SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());

if (OpNo == 2) {
  // The Mask
  EVT DataVT = N->getValueType(0);
  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
} else if (OpNo == 4) {
  // The Index
  if (N->isIndexSigned())
    // Need to sign extend the index since the bits will likely be used.
    NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
  else
    NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
} else
  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));

SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
if (Res == N)
  return SDValue(Res, 0);

// Update triggered CSE, do our own replacement since caller can't.
ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
return SDValue();
2020}

2022SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
                                              unsigned OpNo) {
bool TruncateStore = N->isTruncatingStore();
SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());

if (OpNo == 2) {
  // The Mask
  EVT DataVT = N->getValue().getValueType();
  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
} else if (OpNo == 4) {
  // The Index
  if (N->isIndexSigned())
    // Need to sign extend the index since the bits will likely be used.
    NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
  else
    NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
} else {
  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
  TruncateStore = true;
}

return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
                            SDLoc(N), NewOps, N->getMemOperand(),
                            N->getIndexType(), TruncateStore);
2046}

2048SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
SDValue Op = GetPromotedInteger(N->getOperand(0));
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2051}

2053SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N,
                              ZExtPromotedInteger(N->getOperand(0))), 0);
2056}

2058SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                              ZExtPromotedInteger(N->getOperand(1))), 0);
2061}

2063SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
SDLoc dl(N);
SDValue Op = GetPromotedInteger(N->getOperand(0));
Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2068}

2070SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
assert(OpNo == 2 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo == 2 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo == 2 && \"Don't know how to promote this operand!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2071
, __extension__ __PRETTY_FUNCTION__));

SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
SDValue Carry = N->getOperand(2);
SDLoc DL(N);

Carry = PromoteTargetBoolean(Carry, LHS.getValueType());

return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2081}

2083SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
return SDValue(
    DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2087}

2089SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
// Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
SDValue Op = ZExtPromotedInteger(N->getOperand(0));
return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2093}

2095SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
assert(OpNo > 1 && "Don't know how to promote this operand!")(static_cast <bool> (OpNo > 1 && "Don't know how to promote this operand!"
) ? void (0) : __assert_fail ("OpNo > 1 && \"Don't know how to promote this operand!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2096
, __extension__ __PRETTY_FUNCTION__));
// Promote the rw, locality, and cache type arguments to a supported integer
// width.
SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
                                      Op2, Op3, Op4),
               0);
2105}

2107SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
bool IsStrict = N->isStrictFPOpcode();
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();

// The integer operand is the last operand in FPOWI (so the result and
// floating point operand is already type legalized).

// We can't just promote the exponent type in FPOWI, since we want to lower
// the node to a libcall and we if we promote to a type larger than
// sizeof(int) the libcall might not be according to the targets ABI. Instead
// we rewrite to a libcall here directly, letting makeLibCall handle promotion
// if the target accepts it according to shouldSignExtendTypeInLibCall.
RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected fpowi.") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected fpowi.\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2120
, __extension__ __PRETTY_FUNCTION__));
if (!TLI.getLibcallName(LC)) {
  // Some targets don't have a powi libcall; use pow instead.
  // FIXME: Implement this if some target needs it.
  DAG.getContext()->emitError("Don't know how to promote fpowi to fpow");
  return DAG.getUNDEF(N->getValueType(0));
}
unsigned OpOffset = IsStrict ? 1 : 0;
// The exponent should fit in a sizeof(int) type for the libcall to be valid.
assert(DAG.getLibInfo().getIntSize() ==(static_cast <bool> (DAG.getLibInfo().getIntSize() == N
->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
 "POWI exponent should match with sizeof(int) when doing the libcall."
) ? void (0) : __assert_fail ("DAG.getLibInfo().getIntSize() == N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && \"POWI exponent should match with sizeof(int) when doing the libcall.\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2131
, __extension__ __PRETTY_FUNCTION__))
           N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&(static_cast <bool> (DAG.getLibInfo().getIntSize() == N
->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
 "POWI exponent should match with sizeof(int) when doing the libcall."
) ? void (0) : __assert_fail ("DAG.getLibInfo().getIntSize() == N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && \"POWI exponent should match with sizeof(int) when doing the libcall.\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2131
, __extension__ __PRETTY_FUNCTION__))
       "POWI exponent should match with sizeof(int) when doing the libcall.")(static_cast <bool> (DAG.getLibInfo().getIntSize() == N
->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
 "POWI exponent should match with sizeof(int) when doing the libcall."
) ? void (0) : __assert_fail ("DAG.getLibInfo().getIntSize() == N->getOperand(1 + OpOffset).getValueType().getSizeInBits() && \"POWI exponent should match with sizeof(int) when doing the libcall.\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2131
, __extension__ __PRETTY_FUNCTION__));
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
    DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
ReplaceValueWith(SDValue(N, 0), Tmp.first);
if (IsStrict)
  ReplaceValueWith(SDValue(N, 1), Tmp.second);
return SDValue();
2141}

2143static unsigned getExtendForIntVecReduction(SDNode *N) {
switch (N->getOpcode()) {
default:
  llvm_unreachable("Expected integer vector reduction")::llvm::llvm_unreachable_internal("Expected integer vector reduction"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2146
);
case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VP_REDUCE_ADD:
case ISD::VP_REDUCE_MUL:
case ISD::VP_REDUCE_AND:
case ISD::VP_REDUCE_OR:
case ISD::VP_REDUCE_XOR:
  return ISD::ANY_EXTEND;
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VP_REDUCE_SMAX:
case ISD::VP_REDUCE_SMIN:
  return ISD::SIGN_EXTEND;
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN:
case ISD::VP_REDUCE_UMAX:
case ISD::VP_REDUCE_UMIN:
  return ISD::ZERO_EXTEND;
}
2169}

2171SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
switch (getExtendForIntVecReduction(N)) {
default:
  llvm_unreachable("Impossible extension kind for integer reduction")::llvm::llvm_unreachable_internal("Impossible extension kind for integer reduction"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2174
);
case ISD::ANY_EXTEND:
  return GetPromotedInteger(V);
case ISD::SIGN_EXTEND:
  return SExtPromotedInteger(V);
case ISD::ZERO_EXTEND:
  return ZExtPromotedInteger(V);
}
2182}

2184SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
SDLoc dl(N);
SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));

EVT EltVT = Op.getValueType().getVectorElementType();
EVT VT = N->getValueType(0);

if (VT.bitsGE(EltVT))
  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);

// Result size must be >= element size. If this is not the case after
// promotion, also promote the result type and then truncate.
SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
2198}

2200SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
SDLoc DL(N);
SDValue Op = N->getOperand(OpNo);
SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());

if (OpNo == 2) { // Mask
  // Update in place.
  NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
}

assert(OpNo == 1 && "Unexpected operand for promotion")(static_cast <bool> (OpNo == 1 && "Unexpected operand for promotion"
) ? void (0) : __assert_fail ("OpNo == 1 && \"Unexpected operand for promotion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2211
, __extension__ __PRETTY_FUNCTION__));

Op = PromoteIntOpVectorReduction(N, Op);

NewOps[OpNo] = Op;

EVT VT = N->getValueType(0);
EVT EltVT = Op.getValueType().getScalarType();

if (VT.bitsGE(EltVT))
  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);

// Result size must be >= element/start-value size. If this is not the case
// after promotion, also promote both the start value and result type and
// then truncate.
NewOps[0] =
    DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2230}

2232SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
SDValue Op = ZExtPromotedInteger(N->getOperand(1));
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2235}

2237//===----------------------------------------------------------------------===//
2238//  Integer Result Expansion
2239//===----------------------------------------------------------------------===//

2241/// ExpandIntegerResult - This method is called when the specified result of the
2242/// specified node is found to need expansion.  At this point, the node may also
2243/// have invalid operands or may have other results that need promotion, we just
2244/// know that (at least) one result needs expansion.
2245void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
1
Assuming 'DebugFlag' is false→
2
←
Loop condition is false.  Exiting loop→
           dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer result: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
);
SDValue Lo, Hi;
Lo = Hi = SDValue();

// See if the target wants to custom expand this node.
if (CustomLowerNode(N, N->getValueType(ResNo), true))
3
←
Assuming the condition is false→
4
←
Taking false branch→
  return;

switch (N->getOpcode()) {
5
←
Control jumps to 'case SHL:'  at line 2376→
default:
2257#ifndef NDEBUG
  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
  N->dump(&DAG); dbgs() << "\n";
2260#endif
  report_fatal_error("Do not know how to expand the result of this "
                     "operator!");

case ISD::ARITH_FENCE:  SplitRes_ARITH_FENCE(N, Lo, Hi); break;
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
case ISD::SELECT:       SplitRes_Select(N, Lo, Hi); break;
case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
case ISD::FREEZE:       SplitRes_FREEZE(N, Lo, Hi); break;

case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
case ISD::EXTRACT_ELEMENT:    ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
case ISD::VAARG:              ExpandRes_VAARG(N, Lo, Hi); break;

case ISD::ANY_EXTEND:  ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
case ISD::AssertSext:  ExpandIntRes_AssertSext(N, Lo, Hi); break;
case ISD::AssertZext:  ExpandIntRes_AssertZext(N, Lo, Hi); break;
case ISD::BITREVERSE:  ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
case ISD::BSWAP:       ExpandIntRes_BSWAP(N, Lo, Hi); break;
case ISD::PARITY:      ExpandIntRes_PARITY(N, Lo, Hi); break;
case ISD::Constant:    ExpandIntRes_Constant(N, Lo, Hi); break;
case ISD::ABS:         ExpandIntRes_ABS(N, Lo, Hi); break;
case ISD::CTLZ_ZERO_UNDEF:
case ISD::CTLZ:        ExpandIntRes_CTLZ(N, Lo, Hi); break;
case ISD::CTPOP:       ExpandIntRes_CTPOP(N, Lo, Hi); break;
case ISD::CTTZ_ZERO_UNDEF:
case ISD::CTTZ:        ExpandIntRes_CTTZ(N, Lo, Hi); break;
case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
case ISD::STRICT_FP_TO_SINT:
case ISD::FP_TO_SINT:  ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
case ISD::STRICT_FP_TO_UINT:
case ISD::FP_TO_UINT:  ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
case ISD::FP_TO_SINT_SAT:
case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
case ISD::STRICT_LLROUND:
case ISD::STRICT_LLRINT:
case ISD::LLROUND:
case ISD::LLRINT:      ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
case ISD::LOAD:        ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
case ISD::MUL:         ExpandIntRes_MUL(N, Lo, Hi); break;
case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
case ISD::SDIV:        ExpandIntRes_SDIV(N, Lo, Hi); break;
case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
case ISD::SREM:        ExpandIntRes_SREM(N, Lo, Hi); break;
case ISD::TRUNCATE:    ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;

case ISD::ATOMIC_LOAD_ADD:
case ISD::ATOMIC_LOAD_SUB:
case ISD::ATOMIC_LOAD_AND:
case ISD::ATOMIC_LOAD_CLR:
case ISD::ATOMIC_LOAD_OR:
case ISD::ATOMIC_LOAD_XOR:
case ISD::ATOMIC_LOAD_NAND:
case ISD::ATOMIC_LOAD_MIN:
case ISD::ATOMIC_LOAD_MAX:
case ISD::ATOMIC_LOAD_UMIN:
case ISD::ATOMIC_LOAD_UMAX:
case ISD::ATOMIC_SWAP:
case ISD::ATOMIC_CMP_SWAP: {
  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
  SplitInteger(Tmp.first, Lo, Hi);
  ReplaceValueWith(SDValue(N, 1), Tmp.second);
  break;
}
case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
  AtomicSDNode *AN = cast<AtomicSDNode>(N);
  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
  SDValue Tmp = DAG.getAtomicCmpSwap(
      ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
      N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
      AN->getMemOperand());

  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
  // success simply by comparing the loaded value against the ingoing
  // comparison.
  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
                                 N->getOperand(2), ISD::SETEQ);

  SplitInteger(Tmp, Lo, Hi);
  ReplaceValueWith(SDValue(N, 1), Success);
  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
  break;
}

case ISD::AND:
case ISD::OR:
case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;

case ISD::UMAX:
case ISD::SMAX:
case ISD::UMIN:
case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;

case ISD::ADD:
case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;

case ISD::ADDC:
case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;

case ISD::ADDE:
case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;

case ISD::ADDCARRY:
case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;

case ISD::SADDO_CARRY:
case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;

case ISD::SHL:
case ISD::SRA:
case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
6
←
Calling 'DAGTypeLegalizer::ExpandIntRes_Shift'→

case ISD::SADDO:
case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
case ISD::UADDO:
case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
case ISD::UMULO:
case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;

case ISD::SADDSAT:
case ISD::UADDSAT:
case ISD::SSUBSAT:
case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;

case ISD::SSHLSAT:
case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;

case ISD::SMULFIX:
case ISD::SMULFIXSAT:
case ISD::UMULFIX:
case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;

case ISD::SDIVFIX:
case ISD::SDIVFIXSAT:
case ISD::UDIVFIX:
case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;

case ISD::VECREDUCE_ADD:
case ISD::VECREDUCE_MUL:
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR:
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;

case ISD::ROTL:
case ISD::ROTR:
  ExpandIntRes_Rotate(N, Lo, Hi);
  break;

case ISD::FSHL:
case ISD::FSHR:
  ExpandIntRes_FunnelShift(N, Lo, Hi);
  break;

case ISD::VSCALE:
  ExpandIntRes_VSCALE(N, Lo, Hi);
  break;
}

// If Lo/Hi is null, the sub-method took care of registering results etc.
if (Lo.getNode())
  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2433}

2435/// Lower an atomic node to the appropriate builtin call.
2436std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
unsigned Opc = Node->getOpcode();
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
// Lower to outline atomic libcall if outline atomics enabled,
// or to sync libcall otherwise
RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
EVT RetVT = Node->getValueType(0);
TargetLowering::MakeLibCallOptions CallOptions;
SmallVector<SDValue, 4> Ops;
if (TLI.getLibcallName(LC)) {
  Ops.append(Node->op_begin() + 2, Node->op_end());
  Ops.push_back(Node->getOperand(1));
} else {
  LC = RTLIB::getSYNC(Opc, VT);
  assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected atomic op or value type!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2452
, __extension__ __PRETTY_FUNCTION__))
         "Unexpected atomic op or value type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected atomic op or value type!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2452
, __extension__ __PRETTY_FUNCTION__));
  Ops.append(Node->op_begin() + 1, Node->op_end());
}
return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
                       Node->getOperand(0));
2457}

2459/// N is a shift by a value that needs to be expanded,
2460/// and the shift amount is a constant 'Amt'.  Expand the operation.
2461void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
                                           SDValue &Lo, SDValue &Hi) {
SDLoc DL(N);
// Expand the incoming operand to be shifted, so that we have its parts
SDValue InL, InH;
GetExpandedInteger(N->getOperand(0), InL, InH);

// Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
// splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
if (!Amt) {
  Lo = InL;
  Hi = InH;
  return;
}

EVT NVT = InL.getValueType();
unsigned VTBits = N->getValueType(0).getSizeInBits();
unsigned NVTBits = NVT.getSizeInBits();
EVT ShTy = N->getOperand(1).getValueType();

if (N->getOpcode() == ISD::SHL) {
  if (Amt.uge(VTBits)) {
    Lo = Hi = DAG.getConstant(0, DL, NVT);
  } else if (Amt.ugt(NVTBits)) {
    Lo = DAG.getConstant(0, DL, NVT);
    Hi = DAG.getNode(ISD::SHL, DL,
                     NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
  } else if (Amt == NVTBits) {
    Lo = DAG.getConstant(0, DL, NVT);
    Hi = InL;
  } else {
    Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
    Hi = DAG.getNode(ISD::OR, DL, NVT,
                     DAG.getNode(ISD::SHL, DL, NVT, InH,
                                 DAG.getConstant(Amt, DL, ShTy)),
                     DAG.getNode(ISD::SRL, DL, NVT, InL,
                                 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
  }
  return;
}

if (N->getOpcode() == ISD::SRL) {
  if (Amt.uge(VTBits)) {
    Lo = Hi = DAG.getConstant(0, DL, NVT);
  } else if (Amt.ugt(NVTBits)) {
    Lo = DAG.getNode(ISD::SRL, DL,
                     NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
    Hi = DAG.getConstant(0, DL, NVT);
  } else if (Amt == NVTBits) {
    Lo = InH;
    Hi = DAG.getConstant(0, DL, NVT);
  } else {
    Lo = DAG.getNode(ISD::OR, DL, NVT,
                     DAG.getNode(ISD::SRL, DL, NVT, InL,
                                 DAG.getConstant(Amt, DL, ShTy)),
                     DAG.getNode(ISD::SHL, DL, NVT, InH,
                                 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
    Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
  }
  return;
}

assert(N->getOpcode() == ISD::SRA && "Unknown shift!")(static_cast <bool> (N->getOpcode() == ISD::SRA &&
 "Unknown shift!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SRA && \"Unknown shift!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2523
, __extension__ __PRETTY_FUNCTION__));
if (Amt.uge(VTBits)) {
  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
                        DAG.getConstant(NVTBits - 1, DL, ShTy));
} else if (Amt.ugt(NVTBits)) {
  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
                   DAG.getConstant(Amt - NVTBits, DL, ShTy));
  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
                   DAG.getConstant(NVTBits - 1, DL, ShTy));
} else if (Amt == NVTBits) {
  Lo = InH;
  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
                   DAG.getConstant(NVTBits - 1, DL, ShTy));
} else {
  Lo = DAG.getNode(ISD::OR, DL, NVT,
                   DAG.getNode(ISD::SRL, DL, NVT, InL,
                               DAG.getConstant(Amt, DL, ShTy)),
                   DAG.getNode(ISD::SHL, DL, NVT, InH,
                               DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
}
2544}

2546/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2547/// this shift based on knowledge of the high bit of the shift amount.  If we
2548/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2549/// shift amount.
2550bool DAGTypeLegalizer::
2551ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
SDValue Amt = N->getOperand(1);
10
←
Value assigned to 'Amt.Node'→
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
EVT ShTy = Amt.getValueType();
11
←
Calling 'SDValue::getValueType'→
unsigned ShBits = ShTy.getScalarSizeInBits();
unsigned NVTBits = NVT.getScalarSizeInBits();
assert(isPowerOf2_32(NVTBits) &&(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2558
, __extension__ __PRETTY_FUNCTION__))
       "Expanded integer type size not a power of two!")(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2558
, __extension__ __PRETTY_FUNCTION__));
SDLoc dl(N);

APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
KnownBits Known = DAG.computeKnownBits(N->getOperand(1));

// If we don't know anything about the high bits, exit.
if (((Known.Zero|Known.One) & HighBitMask) == 0)
  return false;

// Get the incoming operand to be shifted.
SDValue InL, InH;
GetExpandedInteger(N->getOperand(0), InL, InH);

// If we know that any of the high bits of the shift amount are one, then we
// can do this as a couple of simple shifts.
if (Known.One.intersects(HighBitMask)) {
  // Mask out the high bit, which we know is set.
  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
                    DAG.getConstant(~HighBitMask, dl, ShTy));

  switch (N->getOpcode()) {
  default: llvm_unreachable("Unknown shift")::llvm::llvm_unreachable_internal("Unknown shift", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2580);
  case ISD::SHL:
    Lo = DAG.getConstant(0, dl, NVT);              // Low part is zero.
    Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
    return true;
  case ISD::SRL:
    Hi = DAG.getConstant(0, dl, NVT);              // Hi part is zero.
    Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
    return true;
  case ISD::SRA:
    Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
                     DAG.getConstant(NVTBits - 1, dl, ShTy));
    Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
    return true;
  }
}

// If we know that all of the high bits of the shift amount are zero, then we
// can do this as a couple of simple shifts.
if (HighBitMask.isSubsetOf(Known.Zero)) {
  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
  // shift if x is zero.  We can use XOR here because x is known to be smaller
  // than 32.
  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
                             DAG.getConstant(NVTBits - 1, dl, ShTy));

  unsigned Op1, Op2;
  switch (N->getOpcode()) {
  default: llvm_unreachable("Unknown shift")::llvm::llvm_unreachable_internal("Unknown shift", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2608);
  case ISD::SHL:  Op1 = ISD::SHL; Op2 = ISD::SRL; break;
  case ISD::SRL:
  case ISD::SRA:  Op1 = ISD::SRL; Op2 = ISD::SHL; break;
  }

  // When shifting right the arithmetic for Lo and Hi is swapped.
  if (N->getOpcode() != ISD::SHL)
    std::swap(InL, InH);

  // Use a little trick to get the bits that move from Lo to Hi. First
  // shift by one bit.
  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
  // Then compute the remaining shift with amount-1.
  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);

  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);

  if (N->getOpcode() != ISD::SHL)
    std::swap(Hi, Lo);
  return true;
}

return false;
2633}

2635/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2636/// of any size.
2637bool DAGTypeLegalizer::
2638ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
SDValue Amt = N->getOperand(1);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
EVT ShTy = Amt.getValueType();
unsigned NVTBits = NVT.getSizeInBits();
assert(isPowerOf2_32(NVTBits) &&(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2644
, __extension__ __PRETTY_FUNCTION__))
       "Expanded integer type size not a power of two!")(static_cast <bool> (isPowerOf2_32(NVTBits) && "Expanded integer type size not a power of two!"
) ? void (0) : __assert_fail ("isPowerOf2_32(NVTBits) && \"Expanded integer type size not a power of two!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2644
, __extension__ __PRETTY_FUNCTION__));
SDLoc dl(N);

// Get the incoming operand to be shifted.
SDValue InL, InH;
GetExpandedInteger(N->getOperand(0), InL, InH);

SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
                               Amt, NVBitsNode, ISD::SETULT);
SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
                              Amt, DAG.getConstant(0, dl, ShTy),
                              ISD::SETEQ);

SDValue LoS, HiS, LoL, HiL;
switch (N->getOpcode()) {
default: llvm_unreachable("Unknown shift")::llvm::llvm_unreachable_internal("Unknown shift", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2662);
case ISD::SHL:
  // Short: ShAmt < NVTBits
  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
  HiS = DAG.getNode(ISD::OR, dl, NVT,
                    DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
                    DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));

  // Long: ShAmt >= NVTBits
  LoL = DAG.getConstant(0, dl, NVT);                    // Lo part is zero.
  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.

  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
  Hi = DAG.getSelect(dl, NVT, isZero, InH,
                     DAG.getSelect(dl, NVT, isShort, HiS, HiL));
  return true;
case ISD::SRL:
  // Short: ShAmt < NVTBits
  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
  LoS = DAG.getNode(ISD::OR, dl, NVT,
                    DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
  // FIXME: If Amt is zero, the following shift generates an undefined result
  // on some architectures.
                    DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));

  // Long: ShAmt >= NVTBits
  HiL = DAG.getConstant(0, dl, NVT);                    // Hi part is zero.
  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.

  Lo = DAG.getSelect(dl, NVT, isZero, InL,
                     DAG.getSelect(dl, NVT, isShort, LoS, LoL));
  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
  return true;
case ISD::SRA:
  // Short: ShAmt < NVTBits
  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
  LoS = DAG.getNode(ISD::OR, dl, NVT,
                    DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
                    DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));

  // Long: ShAmt >= NVTBits
  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
                    DAG.getConstant(NVTBits - 1, dl, ShTy));
  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.

  Lo = DAG.getSelect(dl, NVT, isZero, InL,
                     DAG.getSelect(dl, NVT, isShort, LoS, LoL));
  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
  return true;
}
2712}

2714static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {

switch (Op) {
  default: llvm_unreachable("invalid min/max opcode")::llvm::llvm_unreachable_internal("invalid min/max opcode", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 2717);
  case ISD::SMAX:
    return std::make_pair(ISD::SETGT, ISD::UMAX);
  case ISD::UMAX:
    return std::make_pair(ISD::SETUGT, ISD::UMAX);
  case ISD::SMIN:
    return std::make_pair(ISD::SETLT, ISD::UMIN);
  case ISD::UMIN:
    return std::make_pair(ISD::SETULT, ISD::UMIN);
}
2727}

2729void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
                                         SDValue &Lo, SDValue &Hi) {
SDLoc DL(N);
ISD::NodeType LoOpc;
ISD::CondCode CondC;
std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());

// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);

// Value types
EVT NVT = LHSL.getValueType();
EVT CCT = getSetCCResultType(NVT);

// Hi part is always the same op
Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});

// We need to know whether to select Lo part that corresponds to 'winning'
// Hi part or if Hi parts are equal.
SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);

// Lo part corresponding to the 'winning' Hi part
SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);

// Recursed Lo part if Hi parts are equal, this uses unsigned version
SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});

Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2760}

2762void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
                                         SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);

EVT NVT = LHSL.getValueType();
SDValue LoOps[2] = { LHSL, RHSL };
SDValue HiOps[3] = { LHSH, RHSH };

bool HasOpCarry = TLI.isOperationLegalOrCustom(
    N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
    TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
if (HasOpCarry) {
  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
  if (N->getOpcode() == ISD::ADD) {
    Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
    HiOps[2] = Lo.getValue(1);
    Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
  } else {
    Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
    HiOps[2] = Lo.getValue(1);
    Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
  }
  return;
}

// Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
// them.  TODO: Teach operation legalization how to expand unsupported
// ADDC/ADDE/SUBC/SUBE.  The problem is that these operations generate
// a carry of type MVT::Glue, but there doesn't seem to be any way to
// generate a value of this type in the expanded code sequence.
bool hasCarry =
  TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
                                 ISD::ADDC : ISD::SUBC,
                               TLI.getTypeToExpandTo(*DAG.getContext(), NVT));

if (hasCarry) {
  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
  if (N->getOpcode() == ISD::ADD) {
    Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
    HiOps[2] = Lo.getValue(1);
    Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
  } else {
    Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
    HiOps[2] = Lo.getValue(1);
    Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
  }
  return;
}

bool hasOVF =
  TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
                                 ISD::UADDO : ISD::USUBO,
                               TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);

if (hasOVF) {
  EVT OvfVT = getSetCCResultType(NVT);
  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
  int RevOpc;
  if (N->getOpcode() == ISD::ADD) {
    RevOpc = ISD::SUB;
    Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
    Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
  } else {
    RevOpc = ISD::ADD;
    Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
    Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
  }
  SDValue OVF = Lo.getValue(1);

  switch (BoolType) {
  case TargetLoweringBase::UndefinedBooleanContent:
    OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
    LLVM_FALLTHROUGH[[gnu::fallthrough]];
  case TargetLoweringBase::ZeroOrOneBooleanContent:
    OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
    Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
    break;
  case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
    OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
    Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
  }
  return;
}

if (N->getOpcode() == ISD::ADD) {
  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
                              ISD::SETULT);

  if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
    SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
    Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
    return;
  }

  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
                                 DAG.getConstant(1, dl, NVT),
                                 DAG.getConstant(0, dl, NVT));
  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
                              ISD::SETULT);
  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
                                 DAG.getConstant(1, dl, NVT), Carry1);
  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
} else {
  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
  SDValue Cmp =
    DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
                 LoOps[0], LoOps[1], ISD::SETULT);

  SDValue Borrow;
  if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
    Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
  else
    Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
                           DAG.getConstant(0, dl, NVT));

  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
}
2887}

2889void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
SDValue LoOps[2] = { LHSL, RHSL };
SDValue HiOps[3] = { LHSH, RHSH };

if (N->getOpcode() == ISD::ADDC) {
  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
  HiOps[2] = Lo.getValue(1);
  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
} else {
  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
  HiOps[2] = Lo.getValue(1);
  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
}

// Legalized the flag result - switch anything that used the old flag to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2913}

2915void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
SDValue HiOps[3] = { LHSH, RHSH };

Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
HiOps[2] = Lo.getValue(1);
Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);

// Legalized the flag result - switch anything that used the old flag to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2933}

2935void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
                                           SDValue &Lo, SDValue &Hi) {
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
SDLoc dl(N);

SDValue Ovf;

unsigned CarryOp, NoCarryOp;
ISD::CondCode Cond;
switch(N->getOpcode()) {
  case ISD::UADDO:
    CarryOp = ISD::ADDCARRY;
    NoCarryOp = ISD::ADD;
    Cond = ISD::SETULT;
    break;
  case ISD::USUBO:
    CarryOp = ISD::SUBCARRY;
    NoCarryOp = ISD::SUB;
    Cond = ISD::SETUGT;
    break;
  default:
    llvm_unreachable("Node has unexpected Opcode")::llvm::llvm_unreachable_internal("Node has unexpected Opcode"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 2957
);
}

bool HasCarryOp = TLI.isOperationLegalOrCustom(
    CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));

if (HasCarryOp) {
  // Expand the subcomponents.
  SDValue LHSL, LHSH, RHSL, RHSH;
  GetExpandedInteger(LHS, LHSL, LHSH);
  GetExpandedInteger(RHS, RHSL, RHSH);
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
  SDValue LoOps[2] = { LHSL, RHSL };
  SDValue HiOps[3] = { LHSH, RHSH };

  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
  HiOps[2] = Lo.getValue(1);
  Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);

  Ovf = Hi.getValue(1);
} else {
  // Expand the result by simply replacing it with the equivalent
  // non-overflow-checking operation.
  SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
  SplitInteger(Sum, Lo, Hi);

  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
  // overflows iff a - b > a.
  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
}

// Legalized the flag result - switch anything that used the old flag to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Ovf);
2991}

2993void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
                                              SDValue &Lo, SDValue &Hi) {
// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
SDValue HiOps[3] = { LHSH, RHSH, SDValue() };

Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
HiOps[2] = Lo.getValue(1);
Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);

// Legalized the flag result - switch anything that used the old flag to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3011}

3013void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
                                                 SDValue &Lo, SDValue &Hi) {
// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));

// We need to use an unsigned carry op for the lo part.
unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
                                                      : ISD::SUBCARRY;
Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });

// Legalized the flag result - switch anything that used the old flag to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3031}

3033void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
                                             SDValue &Lo, SDValue &Hi) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
SDValue Op = N->getOperand(0);
if (Op.getValueType().bitsLE(NVT)) {
  // The low part is any extension of the input (which degenerates to a copy).
  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
  Hi = DAG.getUNDEF(NVT);   // The high part is undefined.
} else {
  // For example, extension of an i48 to an i64.  The operand type necessarily
  // promotes to the result type, so will end up being expanded too.
  assert(getTypeAction(Op.getValueType()) ==(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3047
, __extension__ __PRETTY_FUNCTION__))
         TargetLowering::TypePromoteInteger &&(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3047
, __extension__ __PRETTY_FUNCTION__))
         "Only know how to promote this result!")(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3047
, __extension__ __PRETTY_FUNCTION__));
  SDValue Res = GetPromotedInteger(Op);
  assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3050
, __extension__ __PRETTY_FUNCTION__))
         "Operand over promoted?")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3050
, __extension__ __PRETTY_FUNCTION__));
  // Split the promoted operand.  This will simplify when it is expanded.
  SplitInteger(Res, Lo, Hi);
}
3054}

3056void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
                                             SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();
EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
unsigned NVTBits = NVT.getSizeInBits();
unsigned EVTBits = EVT.getSizeInBits();

if (NVTBits < EVTBits) {
  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
                   DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
                                                      EVTBits - NVTBits)));
} else {
  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
  // The high part replicates the sign bit of Lo, make it explicit.
  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
                   DAG.getConstant(NVTBits - 1, dl,
                                   TLI.getPointerTy(DAG.getDataLayout())));
}
3076}

3078void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
                                             SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();
EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
unsigned NVTBits = NVT.getSizeInBits();
unsigned EVTBits = EVT.getSizeInBits();

if (NVTBits < EVTBits) {
  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
                   DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
                                                      EVTBits - NVTBits)));
} else {
  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
  // The high part must be zero, make it explicit.
  Hi = DAG.getConstant(0, dl, NVT);
}
3096}

3098void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
                                             SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3104}

3106void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
                                        SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3112}

3114void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
                                         SDValue &Hi) {
SDLoc dl(N);
// parity(HiLo) -> parity(Lo^Hi)
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();
Lo =
    DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
Hi = DAG.getConstant(0, dl, NVT);
3123}

3125void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
                                           SDValue &Lo, SDValue &Hi) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
unsigned NBitWidth = NVT.getSizeInBits();
auto Constant = cast<ConstantSDNode>(N);
const APInt &Cst = Constant->getAPIntValue();
bool IsTarget = Constant->isTargetOpcode();
bool IsOpaque = Constant->isOpaque();
SDLoc dl(N);
Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
                     IsOpaque);
3137}

3139void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);

SDValue N0 = N->getOperand(0);
GetExpandedInteger(N0, Lo, Hi);
EVT NVT = Lo.getValueType();

// If we have SUBCARRY, use the expanded form of the sra+xor+sub sequence we
// use in LegalizeDAG. The SUB part of the expansion is based on
// ExpandIntRes_ADDSUB which also uses SUBCARRY/USUBO after checking that
// SUBCARRY is LegalOrCustom. Each of the pieces here can be further expanded
// if needed. Shift expansion has a special case for filling with sign bits
// so that we will only end up with one SRA.
bool HasSubCarry = TLI.isOperationLegalOrCustom(
    ISD::SUBCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
if (HasSubCarry) {
  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
  SDValue Sign =
      DAG.getNode(ISD::SRA, dl, NVT, Hi,
                  DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy));
  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
  Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
  Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
  Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
  return;
}

// abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
EVT VT = N->getValueType(0);
SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
                          DAG.getConstant(0, dl, VT), N0);
SDValue NegLo, NegHi;
SplitInteger(Neg, NegLo, NegHi);

SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
                               DAG.getConstant(0, dl, NVT), ISD::SETLT);
Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3178}

3180void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
// ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();

SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
                                 DAG.getConstant(0, dl, NVT), ISD::SETNE);

SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);

Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
                   DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
                               DAG.getConstant(NVT.getSizeInBits(), dl,
                                               NVT)));
Hi = DAG.getConstant(0, dl, NVT);
3198}

3200void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
                                        SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
// ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();
Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
                 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
Hi = DAG.getConstant(0, dl, NVT);
3209}

3211void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
// cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT NVT = Lo.getValueType();

SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
                                 DAG.getConstant(0, dl, NVT), ISD::SETNE);

SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);

Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
                   DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
                               DAG.getConstant(NVT.getSizeInBits(), dl,
                                               NVT)));
Hi = DAG.getConstant(0, dl, NVT);
3229}

3231void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
                                             SDValue &Hi) {
SDLoc dl(N);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
unsigned NBitWidth = NVT.getSizeInBits();

EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
SDValue Chain = Lo.getValue(1);
// The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
                 DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));

// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Chain);
3247}

3249void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
                                             SDValue &Hi) {
SDLoc dl(N);
EVT VT = N->getValueType(0);

bool IsStrict = N->isStrictFPOpcode();
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
SDValue Op = N->getOperand(IsStrict ? 1 : 0);
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
  Op = GetPromotedFloat(Op);

if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
  Op = GetSoftPromotedHalf(Op);
  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
  Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
  SplitInteger(Op, Lo, Hi);
  return;
}

RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected fp-to-sint conversion!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected fp-to-sint conversion!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3270
, __extension__ __PRETTY_FUNCTION__));
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
                                                  CallOptions, dl, Chain);
SplitInteger(Tmp.first, Lo, Hi);

if (IsStrict)
  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3279}

3281void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
                                             SDValue &Hi) {
SDLoc dl(N);
EVT VT = N->getValueType(0);

bool IsStrict = N->isStrictFPOpcode();
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
SDValue Op = N->getOperand(IsStrict ? 1 : 0);
if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
  Op = GetPromotedFloat(Op);

if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
  Op = GetSoftPromotedHalf(Op);
  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
  Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op);
  SplitInteger(Op, Lo, Hi);
  return;
}

RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected fp-to-uint conversion!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected fp-to-uint conversion!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3302
, __extension__ __PRETTY_FUNCTION__));
TargetLowering::MakeLibCallOptions CallOptions;
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
                                                  CallOptions, dl, Chain);
SplitInteger(Tmp.first, Lo, Hi);

if (IsStrict)
  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3310}

3312void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
                                                 SDValue &Hi) {
SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
SplitInteger(Res, Lo, Hi);
3316}

3318void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
                                                 SDValue &Hi) {
SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);

assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&(static_cast <bool> (getTypeAction(Op.getValueType()) !=
 TargetLowering::TypePromoteFloat && "Input type needs to be promoted!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && \"Input type needs to be promoted!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3323
, __extension__ __PRETTY_FUNCTION__))
       "Input type needs to be promoted!")(static_cast <bool> (getTypeAction(Op.getValueType()) !=
 TargetLowering::TypePromoteFloat && "Input type needs to be promoted!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat && \"Input type needs to be promoted!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3323
, __extension__ __PRETTY_FUNCTION__));

EVT VT = Op.getValueType();

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (N->getOpcode() == ISD::LLROUND ||
    N->getOpcode() == ISD::STRICT_LLROUND) {
  if (VT == MVT::f32)
    LC = RTLIB::LLROUND_F32;
  else if (VT == MVT::f64)
    LC = RTLIB::LLROUND_F64;
  else if (VT == MVT::f80)
    LC = RTLIB::LLROUND_F80;
  else if (VT == MVT::f128)
    LC = RTLIB::LLROUND_F128;
  else if (VT == MVT::ppcf128)
    LC = RTLIB::LLROUND_PPCF128;
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected llround input type!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected llround input type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3340
, __extension__ __PRETTY_FUNCTION__));
} else if (N->getOpcode() == ISD::LLRINT ||
           N->getOpcode() == ISD::STRICT_LLRINT) {
  if (VT == MVT::f32)
    LC = RTLIB::LLRINT_F32;
  else if (VT == MVT::f64)
    LC = RTLIB::LLRINT_F64;
  else if (VT == MVT::f80)
    LC = RTLIB::LLRINT_F80;
  else if (VT == MVT::f128)
    LC = RTLIB::LLRINT_F128;
  else if (VT == MVT::ppcf128)
    LC = RTLIB::LLRINT_PPCF128;
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unexpected llrint input type!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected llrint input type!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3353
, __extension__ __PRETTY_FUNCTION__));
} else
  llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3355);

SDLoc dl(N);
EVT RetVT = N->getValueType(0);
SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();

TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
                                                  Op, CallOptions, dl,
                                                  Chain);
SplitInteger(Tmp.first, Lo, Hi);

if (N->isStrictFPOpcode())
  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3370}

3372void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
if (N->isAtomic()) {
  // It's typical to have larger CAS than atomic load instructions.
  SDLoc dl(N);
  EVT VT = N->getMemoryVT();
  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
  SDValue Zero = DAG.getConstant(0, dl, VT);
  SDValue Swap = DAG.getAtomicCmpSwap(
      ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
      VT, VTs, N->getOperand(0),
      N->getOperand(1), Zero, Zero, N->getMemOperand());
  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
  return;
}

if (ISD::isNormalLoad(N)) {
  ExpandRes_NormalLoad(N, Lo, Hi);
  return;
}

assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDLoad(N) &&
 "Indexed load during type legalization!") ? void (0) : __assert_fail
 ("ISD::isUNINDEXEDLoad(N) && \"Indexed load during type legalization!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3394
, __extension__ __PRETTY_FUNCTION__));

EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
SDValue Ch  = N->getChain();
SDValue Ptr = N->getBasePtr();
ISD::LoadExtType ExtType = N->getExtensionType();
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
AAMDNodes AAInfo = N->getAAInfo();
SDLoc dl(N);

assert(NVT.isByteSized() && "Expanded type not byte sized!")(static_cast <bool> (NVT.isByteSized() && "Expanded type not byte sized!"
) ? void (0) : __assert_fail ("NVT.isByteSized() && \"Expanded type not byte sized!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3405
, __extension__ __PRETTY_FUNCTION__));

if (N->getMemoryVT().bitsLE(NVT)) {
  EVT MemVT = N->getMemoryVT();

  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
                      N->getOriginalAlign(), MMOFlags, AAInfo);

  // Remember the chain.
  Ch = Lo.getValue(1);

  if (ExtType == ISD::SEXTLOAD) {
    // The high part is obtained by SRA'ing all but one of the bits of the
    // lo part.
    unsigned LoSize = Lo.getValueSizeInBits();
    Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
                     DAG.getConstant(LoSize - 1, dl,
                                     TLI.getPointerTy(DAG.getDataLayout())));
  } else if (ExtType == ISD::ZEXTLOAD) {
    // The high part is just a zero.
    Hi = DAG.getConstant(0, dl, NVT);
  } else {
    assert(ExtType == ISD::EXTLOAD && "Unknown extload!")(static_cast <bool> (ExtType == ISD::EXTLOAD &&
 "Unknown extload!") ? void (0) : __assert_fail ("ExtType == ISD::EXTLOAD && \"Unknown extload!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3427
, __extension__ __PRETTY_FUNCTION__));
    // The high part is undefined.
    Hi = DAG.getUNDEF(NVT);
  }
} else if (DAG.getDataLayout().isLittleEndian()) {
  // Little-endian - low bits are at low addresses.
  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
                   N->getOriginalAlign(), MMOFlags, AAInfo);

  unsigned ExcessBits =
    N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);

  // Increment the pointer to the other half.
  unsigned IncrementSize = NVT.getSizeInBits()/8;
  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
                      N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
                      N->getOriginalAlign(), MMOFlags, AAInfo);

  // Build a factor node to remember that this load is independent of the
  // other one.
  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
                   Hi.getValue(1));
} else {
  // Big-endian - high bits are at low addresses.  Favor aligned loads at
  // the cost of some bit-fiddling.
  EVT MemVT = N->getMemoryVT();
  unsigned EBytes = MemVT.getStoreSize();
  unsigned IncrementSize = NVT.getSizeInBits()/8;
  unsigned ExcessBits = (EBytes - IncrementSize)*8;

  // Load both the high bits and maybe some of the low bits.
  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
                      EVT::getIntegerVT(*DAG.getContext(),
                                        MemVT.getSizeInBits() - ExcessBits),
                      N->getOriginalAlign(), MMOFlags, AAInfo);

  // Increment the pointer to the other half.
  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
  // Load the rest of the low bits.
  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
                      N->getPointerInfo().getWithOffset(IncrementSize),
                      EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
                      N->getOriginalAlign(), MMOFlags, AAInfo);

  // Build a factor node to remember that this load is independent of the
  // other one.
  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
                   Hi.getValue(1));

  if (ExcessBits < NVT.getSizeInBits()) {
    // Transfer low bits from the bottom of Hi to the top of Lo.
    Lo = DAG.getNode(
        ISD::OR, dl, NVT, Lo,
        DAG.getNode(ISD::SHL, dl, NVT, Hi,
                    DAG.getConstant(ExcessBits, dl,
                                    TLI.getPointerTy(DAG.getDataLayout()))));
    // Move high bits to the right position in Hi.
    Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
                     Hi,
                     DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
                                     TLI.getPointerTy(DAG.getDataLayout())));
  }
}

// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Ch);
3496}

3498void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
                                          SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
SDValue LL, LH, RL, RH;
GetExpandedInteger(N->getOperand(0), LL, LH);
GetExpandedInteger(N->getOperand(1), RL, RH);
Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3506}

3508void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
                                      SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
SDLoc dl(N);

SDValue LL, LH, RL, RH;
GetExpandedInteger(N->getOperand(0), LL, LH);
GetExpandedInteger(N->getOperand(1), RL, RH);

if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
                  TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
                  LL, LH, RL, RH))
  return;

// If nothing else, we can make a libcall.
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (VT == MVT::i16)
  LC = RTLIB::MUL_I16;
else if (VT == MVT::i32)
  LC = RTLIB::MUL_I32;
else if (VT == MVT::i64)
  LC = RTLIB::MUL_I64;
else if (VT == MVT::i128)
  LC = RTLIB::MUL_I128;

if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
  // We'll expand the multiplication by brute force because we have no other
  // options. This is a trivially-generalized version of the code from
  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
  // 4.3.1).
  unsigned Bits = NVT.getSizeInBits();
  unsigned HalfBits = Bits >> 1;
  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
                                 NVT);
  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);

  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);

  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);

  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
                          DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);

  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
                          DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);

  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
                          DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
                          DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
                   DAG.getNode(ISD::SHL, dl, NVT, V, Shift));

  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
                   DAG.getNode(ISD::ADD, dl, NVT,
                               DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
                               DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
  return;
}

SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
             Lo, Hi);
3582}

3584void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
                                                   SDValue &Hi) {
SDLoc DL(N);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
Lo = R.getValue(0);
Hi = R.getValue(1);
ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3593}

3595void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
                                            SDValue &Hi) {
SDValue Result = TLI.expandAddSubSat(N, DAG);
SplitInteger(Result, Lo, Hi);
3599}

3601void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
                                         SDValue &Hi) {
SDValue Result = TLI.expandShlSat(N, DAG);
SplitInteger(Result, Lo, Hi);
3605}

3607/// This performs an expansion of the integer result for a fixed point
3608/// multiplication. The default expansion performs rounding down towards
3609/// negative infinity, though targets that do care about rounding should specify
3610/// a target hook for rounding and provide their own expansion or lowering of
3611/// fixed point multiplication to be consistent with rounding.
3612void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
                                         SDValue &Hi) {
SDLoc dl(N);
EVT VT = N->getValueType(0);
unsigned VTSize = VT.getScalarSizeInBits();
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
uint64_t Scale = N->getConstantOperandVal(2);
bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
                   N->getOpcode() == ISD::UMULFIXSAT);
bool Signed = (N->getOpcode() == ISD::SMULFIX ||
               N->getOpcode() == ISD::SMULFIXSAT);

// Handle special case when scale is equal to zero.
if (!Scale) {
  SDValue Result;
  if (!Saturating) {
    Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
  } else {
    EVT BoolVT = getSetCCResultType(VT);
    unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
    Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
    SDValue Product = Result.getValue(0);
    SDValue Overflow = Result.getValue(1);
    if (Signed) {
      APInt MinVal = APInt::getSignedMinValue(VTSize);
      APInt MaxVal = APInt::getSignedMaxValue(VTSize);
      SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
      SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
      SDValue Zero = DAG.getConstant(0, dl, VT);
      // Xor the inputs, if resulting sign bit is 0 the product will be
      // positive, else negative.
      SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
      SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
      Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
      Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
    } else {
      // For unsigned multiplication, we only need to check the max since we
      // can't really overflow towards zero.
      APInt MaxVal = APInt::getMaxValue(VTSize);
      SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
      Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
    }
  }
  SplitInteger(Result, Lo, Hi);
  return;
}

// For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
// cover for unhandled cases below, while still being valid for UMULFIX[SAT].
assert(Scale <= VTSize && "Scale can't be larger than the value type size.")(static_cast <bool> (Scale <= VTSize && "Scale can't be larger than the value type size."
) ? void (0) : __assert_fail ("Scale <= VTSize && \"Scale can't be larger than the value type size.\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3662
, __extension__ __PRETTY_FUNCTION__));

EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
SDValue LL, LH, RL, RH;
GetExpandedInteger(LHS, LL, LH);
GetExpandedInteger(RHS, RL, RH);
SmallVector<SDValue, 4> Result;

unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
                        TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
                        LL, LH, RL, RH)) {
  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
  return;
}

unsigned NVTSize = NVT.getScalarSizeInBits();
assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "(static_cast <bool> ((VTSize == NVTSize * 2) &&
 "Expected the new value type to be half " "the size of the current value type"
) ? void (0) : __assert_fail ("(VTSize == NVTSize * 2) && \"Expected the new value type to be half \" \"the size of the current value type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3680
, __extension__ __PRETTY_FUNCTION__))
                                  "the size of the current value type")(static_cast <bool> ((VTSize == NVTSize * 2) &&
 "Expected the new value type to be half " "the size of the current value type"
) ? void (0) : __assert_fail ("(VTSize == NVTSize * 2) && \"Expected the new value type to be half \" \"the size of the current value type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3680
, __extension__ __PRETTY_FUNCTION__));
EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());

// After getting the multiplication result in 4 parts, we need to perform a
// shift right by the amount of the scale to get the result in that scale.
//
// Let's say we multiply 2 64 bit numbers. The resulting value can be held in
// 128 bits that are cut into 4 32-bit parts:
//
//      HH       HL       LH       LL
//  |---32---|---32---|---32---|---32---|
// 128      96       64       32        0
//
//                    |------VTSize-----|
//
//                             |NVTSize-|
//
// The resulting Lo and Hi would normally be in LL and LH after the shift. But
// to avoid unneccessary shifting of all 4 parts, we can adjust the shift
// amount and get Lo and Hi using two funnel shifts. Or for the special case
// when Scale is a multiple of NVTSize we can just pick the result without
// shifting.
uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
if (Scale % NVTSize) {
  SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
  Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
                   ShiftAmount);
  Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
                   ShiftAmount);
} else {
  Lo = Result[Part0];
  Hi = Result[Part0 + 1];
}

// Unless saturation is requested we are done. The result is in <Hi,Lo>.
if (!Saturating)
  return;

// Can not overflow when there is no integer part.
if (Scale == VTSize)
  return;

// To handle saturation we must check for overflow in the multiplication.
//
// Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
// aren't all zeroes.
//
// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
// aren't all ones or all zeroes.
//
// We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
// highest bit of HH determines saturation direction in the event of signed
// saturation.

SDValue ResultHL = Result[2];
SDValue ResultHH = Result[3];

SDValue SatMax, SatMin;
SDValue NVTZero = DAG.getConstant(0, dl, NVT);
SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
EVT BoolNVT = getSetCCResultType(NVT);

if (!Signed) {
  if (Scale < NVTSize) {
    // Overflow happened if ((HH | (HL >> Scale)) != 0).
    SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
                                     DAG.getConstant(Scale, dl, ShiftTy));
    SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
    SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
  } else if (Scale == NVTSize) {
    // Overflow happened if (HH != 0).
    SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
  } else if (Scale < VTSize) {
    // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
    SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
                                     DAG.getConstant(Scale - NVTSize, dl,
                                                     ShiftTy));
    SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
  } else
    llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"::llvm::llvm_unreachable_internal("Scale must be less or equal to VTSize for UMULFIXSAT"
 "(and saturation can't happen with Scale==VTSize).", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3760)
                     "(and saturation can't happen with Scale==VTSize).")::llvm::llvm_unreachable_internal("Scale must be less or equal to VTSize for UMULFIXSAT"
 "(and saturation can't happen with Scale==VTSize).", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 3760);

  Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
  Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
  return;
}

if (Scale < NVTSize) {
  // The number of overflow bits we can check are VTSize - Scale + 1 (we
  // include the sign bit). If these top bits are > 0, then we overflowed past
  // the max value. If these top bits are < -1, then we overflowed past the
  // min value. Otherwise, we did not overflow.
  unsigned OverflowBits = VTSize - Scale + 1;
  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&(static_cast <bool> (OverflowBits <= VTSize &&
 OverflowBits > NVTSize && "Extent of overflow bits must start within HL"
) ? void (0) : __assert_fail ("OverflowBits <= VTSize && OverflowBits > NVTSize && \"Extent of overflow bits must start within HL\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3774
, __extension__ __PRETTY_FUNCTION__))
         "Extent of overflow bits must start within HL")(static_cast <bool> (OverflowBits <= VTSize &&
 OverflowBits > NVTSize && "Extent of overflow bits must start within HL"
) ? void (0) : __assert_fail ("OverflowBits <= VTSize && OverflowBits > NVTSize && \"Extent of overflow bits must start within HL\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3774
, __extension__ __PRETTY_FUNCTION__));
  SDValue HLHiMask = DAG.getConstant(
      APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
  SDValue HLLoMask = DAG.getConstant(
      APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
  // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
  SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
                       DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
  // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
  SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
                       DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
} else if (Scale == NVTSize) {
  // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
                       DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
  // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
                       DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
} else if (Scale < VTSize) {
  // This is similar to the case when we saturate if Scale < NVTSize, but we
  // only need to check HH.
  unsigned OverflowBits = VTSize - Scale + 1;
  SDValue HHHiMask = DAG.getConstant(
      APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
  SDValue HHLoMask = DAG.getConstant(
      APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
} else
  llvm_unreachable("Illegal scale for signed fixed point mul.")::llvm::llvm_unreachable_internal("Illegal scale for signed fixed point mul."
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3815
);

// Saturate to signed maximum.
APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
APInt MaxLo = APInt::getAllOnes(NVTSize);
Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
// Saturate to signed minimum.
APInt MinHi = APInt::getSignedMinValue(NVTSize);
Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3826}

3828void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
                                         SDValue &Hi) {
SDLoc dl(N);
// Try expanding in the existing type first.
SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
                                      N->getOperand(1),
                                      N->getConstantOperandVal(2), DAG);

if (!Res)
  Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
                          N->getConstantOperandVal(2), TLI, DAG);
SplitInteger(Res, Lo, Hi);
3840}

3842void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
                                           SDValue &Lo, SDValue &Hi) {
assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&(static_cast <bool> ((Node->getOpcode() == ISD::SADDO
 || Node->getOpcode() == ISD::SSUBO) && "Node has unexpected Opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && \"Node has unexpected Opcode\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3845
, __extension__ __PRETTY_FUNCTION__))
       "Node has unexpected Opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SADDO
 || Node->getOpcode() == ISD::SSUBO) && "Node has unexpected Opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) && \"Node has unexpected Opcode\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3845
, __extension__ __PRETTY_FUNCTION__));
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
SDLoc dl(Node);

SDValue Ovf;

bool IsAdd = Node->getOpcode() == ISD::SADDO;
unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;

bool HasCarryOp = TLI.isOperationLegalOrCustom(
    CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));

if (HasCarryOp) {
  // Expand the subcomponents.
  SDValue LHSL, LHSH, RHSL, RHSH;
  GetExpandedInteger(LHS, LHSL, LHSH);
  GetExpandedInteger(RHS, RHSL, RHSH);
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));

  Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
  Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });

  Ovf = Hi.getValue(1);
} else {
  // Expand the result by simply replacing it with the equivalent
  // non-overflow-checking operation.
  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
                            ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
                            LHS, RHS);
  SplitInteger(Sum, Lo, Hi);

  // Compute the overflow.
  //
  //   LHSSign -> LHS < 0
  //   RHSSign -> RHS < 0
  //   SumSign -> Sum < 0
  //
  //   Add:
  //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
  //   Sub:
  //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
  //
  // To get better codegen we can rewrite this by doing bitwise math on
  // the integers and extract the final sign bit at the end. So the
  // above becomes:
  //
  //   Add:
  //   Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
  //   Sub:
  //   Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
  //
  // NOTE: This is different than the expansion we do in expandSADDSUBO
  // because it is more costly to determine the RHS is > 0 for SSUBO with the
  // integers split.
  EVT VT = LHS.getValueType();
  SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
  if (IsAdd)
    SignsMatch = DAG.getNOT(dl, SignsMatch, VT);

  SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
  Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
  EVT OType = Node->getValueType(1);
  Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
}

// Use the calculated overflow everywhere.
ReplaceValueWith(SDValue(Node, 1), Ovf);
3913}

3915// Emit a call to __udivei4 and friends which require
3916// the arguments be based on the stack
3917// and extra argument that contains the number of bits of the operands.
3918// Returns the result of the call operation.
3919static SDValue ExpandExtIntRes_DIVREM(const TargetLowering &TLI,
                                    const RTLIB::Libcall &LC,
                                    SelectionDAG &DAG, SDNode *N,
                                    const SDLoc &DL, const EVT &VT) {

SDValue InChain = DAG.getEntryNode();

TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;

// The signature of __udivei4 is
// void __udivei4(unsigned int *quo, unsigned int *a, unsigned int *b,
// unsigned int bits)
EVT ArgVT = N->op_begin()->getValueType();
assert(ArgVT.isInteger() && ArgVT.getSizeInBits() > 128 &&(static_cast <bool> (ArgVT.isInteger() && ArgVT
.getSizeInBits() > 128 && "Unexpected argument type for lowering"
) ? void (0) : __assert_fail ("ArgVT.isInteger() && ArgVT.getSizeInBits() > 128 && \"Unexpected argument type for lowering\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3934
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected argument type for lowering")(static_cast <bool> (ArgVT.isInteger() && ArgVT
.getSizeInBits() > 128 && "Unexpected argument type for lowering"
) ? void (0) : __assert_fail ("ArgVT.isInteger() && ArgVT.getSizeInBits() > 128 && \"Unexpected argument type for lowering\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 3934
, __extension__ __PRETTY_FUNCTION__));
Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());

SDValue Output = DAG.CreateStackTemporary(ArgVT);
Entry.Node = Output;
Entry.Ty = ArgTy->getPointerTo();
Entry.IsSExt = false;
Entry.IsZExt = false;
Args.push_back(Entry);

for (const llvm::SDUse &Op : N->ops()) {
  SDValue StackPtr = DAG.CreateStackTemporary(ArgVT);
  InChain = DAG.getStore(InChain, DL, Op, StackPtr, MachinePointerInfo());
  Entry.Node = StackPtr;
  Entry.Ty = ArgTy->getPointerTo();
  Entry.IsSExt = false;
  Entry.IsZExt = false;
  Args.push_back(Entry);
}

int Bits = N->getOperand(0)
               .getValueType()
               .getTypeForEVT(*DAG.getContext())
               ->getIntegerBitWidth();
Entry.Node = DAG.getConstant(Bits, DL, TLI.getPointerTy(DAG.getDataLayout()));
Entry.Ty = Type::getInt32Ty(*DAG.getContext());
Entry.IsSExt = false;
Entry.IsZExt = true;
Args.push_back(Entry);

SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
                                       TLI.getPointerTy(DAG.getDataLayout()));

TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(DL)
    .setChain(InChain)
    .setLibCallee(TLI.getLibcallCallingConv(LC),
                  Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args))
    .setDiscardResult();

SDValue Chain = TLI.LowerCallTo(CLI).second;

return DAG.getLoad(ArgVT, DL, Chain, Output, MachinePointerInfo());
3977}

3979void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
SDLoc dl(N);
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };

if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
  SplitInteger(Res.getValue(0), Lo, Hi);
  return;
}

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (VT == MVT::i16)
  LC = RTLIB::SDIV_I16;
else if (VT == MVT::i32)
  LC = RTLIB::SDIV_I32;
else if (VT == MVT::i64)
  LC = RTLIB::SDIV_I64;
else if (VT == MVT::i128)
  LC = RTLIB::SDIV_I128;

else {
  SDValue Result =
      ExpandExtIntRes_DIVREM(TLI, RTLIB::SDIV_IEXT, DAG, N, dl, VT);
  SplitInteger(Result, Lo, Hi);
  return;
}

assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unsupported SDIV!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported SDIV!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4008
, __extension__ __PRETTY_FUNCTION__));

TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4013}

4015void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
                                        SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
SDLoc dl(N);

// If we can emit an efficient shift operation, do so now.  Check to see if
// the RHS is a constant.
if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
7
←
Assuming 'CN' is null→
8
←
Taking false branch→
  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);

// If we can determine that the high bit of the shift is zero or one, even if
// the low bits are variable, emit this shift in an optimized form.
if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
9
←
Calling 'DAGTypeLegalizer::ExpandShiftWithKnownAmountBit'→
  return;

// If this target supports shift_PARTS, use it.  First, map to the _PARTS opc.
unsigned PartsOpc;
if (N->getOpcode() == ISD::SHL) {
  PartsOpc = ISD::SHL_PARTS;
} else if (N->getOpcode() == ISD::SRL) {
  PartsOpc = ISD::SRL_PARTS;
} else {
  assert(N->getOpcode() == ISD::SRA && "Unknown shift!")(static_cast <bool> (N->getOpcode() == ISD::SRA &&
 "Unknown shift!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SRA && \"Unknown shift!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4037
, __extension__ __PRETTY_FUNCTION__));
  PartsOpc = ISD::SRA_PARTS;
}

// Next check to see if the target supports this SHL_PARTS operation or if it
// will custom expand it. Don't lower this to SHL_PARTS when we optimise for
// size, but create a libcall instead.
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
const bool LegalOrCustom =
  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
  Action == TargetLowering::Custom;

if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
  // Expand the subcomponents.
  SDValue LHSL, LHSH;
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
  EVT VT = LHSL.getValueType();

  // If the shift amount operand is coming from a vector legalization it may
  // have an illegal type.  Fix that first by casting the operand, otherwise
  // the new SHL_PARTS operation would need further legalization.
  SDValue ShiftOp = N->getOperand(1);
  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
  if (ShiftOp.getValueType() != ShiftTy)
    ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);

  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
  Hi = Lo.getValue(1);
  return;
}

// Otherwise, emit a libcall.
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
bool isSigned;
if (N->getOpcode() == ISD::SHL) {
  isSigned = false; /*sign irrelevant*/
  if (VT == MVT::i16)
    LC = RTLIB::SHL_I16;
  else if (VT == MVT::i32)
    LC = RTLIB::SHL_I32;
  else if (VT == MVT::i64)
    LC = RTLIB::SHL_I64;
  else if (VT == MVT::i128)
    LC = RTLIB::SHL_I128;
} else if (N->getOpcode() == ISD::SRL) {
  isSigned = false;
  if (VT == MVT::i16)
    LC = RTLIB::SRL_I16;
  else if (VT == MVT::i32)
    LC = RTLIB::SRL_I32;
  else if (VT == MVT::i64)
    LC = RTLIB::SRL_I64;
  else if (VT == MVT::i128)
    LC = RTLIB::SRL_I128;
} else {
  assert(N->getOpcode() == ISD::SRA && "Unknown shift!")(static_cast <bool> (N->getOpcode() == ISD::SRA &&
 "Unknown shift!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::SRA && \"Unknown shift!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4094
, __extension__ __PRETTY_FUNCTION__));
  isSigned = true;
  if (VT == MVT::i16)
    LC = RTLIB::SRA_I16;
  else if (VT == MVT::i32)
    LC = RTLIB::SRA_I32;
  else if (VT == MVT::i64)
    LC = RTLIB::SRA_I64;
  else if (VT == MVT::i128)
    LC = RTLIB::SRA_I128;
}

if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
  EVT ShAmtTy =
      EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
  SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
  SDValue Ops[2] = {N->getOperand(0), ShAmt};
  TargetLowering::MakeLibCallOptions CallOptions;
  CallOptions.setSExt(isSigned);
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
  return;
}

if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
  llvm_unreachable("Unsupported shift!")::llvm::llvm_unreachable_internal("Unsupported shift!", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4118);
4119}

4121void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
                                              SDValue &Lo, SDValue &Hi) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
SDValue Op = N->getOperand(0);
if (Op.getValueType().bitsLE(NVT)) {
  // The low part is sign extension of the input (degenerates to a copy).
  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
  // The high part is obtained by SRA'ing all but one of the bits of low part.
  unsigned LoSize = NVT.getSizeInBits();
  Hi = DAG.getNode(
      ISD::SRA, dl, NVT, Lo,
      DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
} else {
  // For example, extension of an i48 to an i64.  The operand type necessarily
  // promotes to the result type, so will end up being expanded too.
  assert(getTypeAction(Op.getValueType()) ==(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4139
, __extension__ __PRETTY_FUNCTION__))
         TargetLowering::TypePromoteInteger &&(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4139
, __extension__ __PRETTY_FUNCTION__))
         "Only know how to promote this result!")(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4139
, __extension__ __PRETTY_FUNCTION__));
  SDValue Res = GetPromotedInteger(Op);
  assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4142
, __extension__ __PRETTY_FUNCTION__))
         "Operand over promoted?")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4142
, __extension__ __PRETTY_FUNCTION__));
  // Split the promoted operand.  This will simplify when it is expanded.
  SplitInteger(Res, Lo, Hi);
  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
                   DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
                                                      ExcessBits)));
}
4150}

4152void DAGTypeLegalizer::
4153ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
GetExpandedInteger(N->getOperand(0), Lo, Hi);
EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();

if (EVT.bitsLE(Lo.getValueType())) {
  // sext_inreg the low part if needed.
  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
                   N->getOperand(1));

  // The high part gets the sign extension from the lo-part.  This handles
  // things like sextinreg V:i64 from i8.
  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
                   DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
                                   TLI.getPointerTy(DAG.getDataLayout())));
} else {
  // For example, extension of an i48 to an i64.  Leave the low part alone,
  // sext_inreg the high part.
  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
                   DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
                                                      ExcessBits)));
}
4176}

4178void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
SDLoc dl(N);
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };

if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
  SplitInteger(Res.getValue(1), Lo, Hi);
  return;
}

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (VT == MVT::i16)
  LC = RTLIB::SREM_I16;
else if (VT == MVT::i32)
  LC = RTLIB::SREM_I32;
else if (VT == MVT::i64)
  LC = RTLIB::SREM_I64;
else if (VT == MVT::i128)
  LC = RTLIB::SREM_I128;

else {
  SDValue Result =
      ExpandExtIntRes_DIVREM(TLI, RTLIB::SREM_IEXT, DAG, N, dl, VT);
  SplitInteger(Result, Lo, Hi);
  return;
}

assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unsupported SREM!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported SREM!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4207
, __extension__ __PRETTY_FUNCTION__));

TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4212}

4214void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
                                           SDValue &Lo, SDValue &Hi) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
                 N->getOperand(0),
                 DAG.getConstant(NVT.getSizeInBits(), dl,
                                 TLI.getPointerTy(DAG.getDataLayout())));
Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
4224}

4226void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
                                        SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
SDLoc dl(N);

if (N->getOpcode() == ISD::UMULO) {
  // This section expands the operation into the following sequence of
  // instructions. `iNh` here refers to a type which has half the bit width of
  // the type the original operation operated on.
  //
  // %0 = %LHS.HI != 0 && %RHS.HI != 0
  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
  // %4 = add iNh %1.0, %2.0 as iN
  // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
  //
  // %lo = %3.LO
  // %hi = %5.0
  // %ovf = %0 || %1.1 || %2.1 || %5.1
  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
  GetExpandedInteger(LHS, LHSLow, LHSHigh);
  GetExpandedInteger(RHS, RHSLow, RHSHigh);
  EVT HalfVT = LHSLow.getValueType();
  EVT BitVT = N->getValueType(1);
  SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);

  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
    DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
    DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));

  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));

  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));

  SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);

  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
  // operation recursively legalized?).
  //
  // Many backends understand this pattern and will convert into LOHI
  // themselves, if applicable.
  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
    DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
    DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
  SplitInteger(Three, Lo, Hi);

  Hi = DAG.getNode(ISD::UADDO, dl, VTHalfWithO, Hi, HighSum);
  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Hi.getValue(1));
  ReplaceValueWith(SDValue(N, 1), Overflow);
  return;
}

Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());

// Replace this with a libcall that will check overflow.
RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (VT == MVT::i32)
  LC = RTLIB::MULO_I32;
else if (VT == MVT::i64)
  LC = RTLIB::MULO_I64;
else if (VT == MVT::i128)
  LC = RTLIB::MULO_I128;

if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
  // FIXME: This is not an optimal expansion, but better than crashing.
  EVT WideVT =
      EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
  SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(0));
  SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, dl, WideVT, N->getOperand(1));
  SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
  SDValue MulLo, MulHi;
  SplitInteger(Mul, MulLo, MulHi);
  SDValue SRA =
      DAG.getNode(ISD::SRA, dl, VT, MulLo,
                  DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, VT));
  SDValue Overflow =
      DAG.getSetCC(dl, N->getValueType(1), MulHi, SRA, ISD::SETNE);
  SplitInteger(MulLo, Lo, Hi);
  ReplaceValueWith(SDValue(N, 1), Overflow);
  return;
}

SDValue Temp = DAG.CreateStackTemporary(PtrVT);
// Temporary for the overflow value, default it to zero.
SDValue Chain =
    DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
                 MachinePointerInfo());

TargetLowering::ArgListTy Args;
TargetLowering::ArgListEntry Entry;
for (const SDValue &Op : N->op_values()) {
  EVT ArgVT = Op.getValueType();
  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
  Entry.Node = Op;
  Entry.Ty = ArgTy;
  Entry.IsSExt = true;
  Entry.IsZExt = false;
  Args.push_back(Entry);
}

// Also pass the address of the overflow check.
Entry.Node = Temp;
Entry.Ty = PtrTy->getPointerTo();
Entry.IsSExt = true;
Entry.IsZExt = false;
Args.push_back(Entry);

SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);

TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(dl)
    .setChain(Chain)
    .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
    .setSExtResult();

std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);

SplitInteger(CallInfo.first, Lo, Hi);
SDValue Temp2 =
    DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
                           DAG.getConstant(0, dl, PtrVT),
                           ISD::SETNE);
// Use the overflow from the libcall everywhere.
ReplaceValueWith(SDValue(N, 1), Ofl);
4359}

4361void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
SDLoc dl(N);
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };

if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
  SplitInteger(Res.getValue(0), Lo, Hi);
  return;
}

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (VT == MVT::i16)
  LC = RTLIB::UDIV_I16;
else if (VT == MVT::i32)
  LC = RTLIB::UDIV_I32;
else if (VT == MVT::i64)
  LC = RTLIB::UDIV_I64;
else if (VT == MVT::i128)
  LC = RTLIB::UDIV_I128;

else {
  SDValue Result =
      ExpandExtIntRes_DIVREM(TLI, RTLIB::UDIV_IEXT, DAG, N, dl, VT);
  SplitInteger(Result, Lo, Hi);
  return;
}

assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unsupported UDIV!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported UDIV!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4390
, __extension__ __PRETTY_FUNCTION__));

TargetLowering::MakeLibCallOptions CallOptions;
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4394}

4396void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
                                       SDValue &Lo, SDValue &Hi) {
EVT VT = N->getValueType(0);
SDLoc dl(N);
SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };

if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
  SplitInteger(Res.getValue(1), Lo, Hi);
  return;
}

RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
if (VT == MVT::i16)
  LC = RTLIB::UREM_I16;
else if (VT == MVT::i32)
  LC = RTLIB::UREM_I32;
else if (VT == MVT::i64)
  LC = RTLIB::UREM_I64;
else if (VT == MVT::i128)
  LC = RTLIB::UREM_I128;

else {
  SDValue Result =
      ExpandExtIntRes_DIVREM(TLI, RTLIB::UREM_IEXT, DAG, N, dl, VT);
  SplitInteger(Result, Lo, Hi);
  return;
}

assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Unsupported UREM!") ? void (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unsupported UREM!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4425
, __extension__ __PRETTY_FUNCTION__));

TargetLowering::MakeLibCallOptions CallOptions;
SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4429}

4431void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
                                              SDValue &Lo, SDValue &Hi) {
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDLoc dl(N);
SDValue Op = N->getOperand(0);
if (Op.getValueType().bitsLE(NVT)) {
  // The low part is zero extension of the input (degenerates to a copy).
  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
  Hi = DAG.getConstant(0, dl, NVT);   // The high part is just a zero.
} else {
  // For example, extension of an i48 to an i64.  The operand type necessarily
  // promotes to the result type, so will end up being expanded too.
  assert(getTypeAction(Op.getValueType()) ==(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4445
, __extension__ __PRETTY_FUNCTION__))
         TargetLowering::TypePromoteInteger &&(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4445
, __extension__ __PRETTY_FUNCTION__))
         "Only know how to promote this result!")(static_cast <bool> (getTypeAction(Op.getValueType()) ==
 TargetLowering::TypePromoteInteger && "Only know how to promote this result!"
) ? void (0) : __assert_fail ("getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger && \"Only know how to promote this result!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4445
, __extension__ __PRETTY_FUNCTION__));
  SDValue Res = GetPromotedInteger(Op);
  assert(Res.getValueType() == N->getValueType(0) &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4448
, __extension__ __PRETTY_FUNCTION__))
         "Operand over promoted?")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && "Operand over promoted?") ? void (0) : __assert_fail
 ("Res.getValueType() == N->getValueType(0) && \"Operand over promoted?\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4448
, __extension__ __PRETTY_FUNCTION__));
  // Split the promoted operand.  This will simplify when it is expanded.
  SplitInteger(Res, Lo, Hi);
  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
  Hi = DAG.getZeroExtendInReg(Hi, dl,
                              EVT::getIntegerVT(*DAG.getContext(),
                                                ExcessBits));
}
4456}

4458void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
                                              SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
SDValue Zero = DAG.getConstant(0, dl, VT);
SDValue Swap = DAG.getAtomicCmpSwap(
    ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
    cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
    N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());

ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
4471}

4473void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
                                            SDValue &Lo, SDValue &Hi) {
// TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
// both halves independently.
SDValue Res = TLI.expandVecReduce(N, DAG);
SplitInteger(Res, Lo, Hi);
4479}

4481void DAGTypeLegalizer::ExpandIntRes_Rotate(SDNode *N,
                                         SDValue &Lo, SDValue &Hi) {
// Delegate to funnel-shift expansion.
SDLoc DL(N);
unsigned Opcode = N->getOpcode() == ISD::ROTL ? ISD::FSHL : ISD::FSHR;
SDValue Res = DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(0),
                          N->getOperand(0), N->getOperand(1));
SplitInteger(Res, Lo, Hi);
4489}

4491void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
                                              SDValue &Hi) {
// Values numbered from least significant to most significant.
SDValue In1, In2, In3, In4;
GetExpandedInteger(N->getOperand(0), In3, In4);
GetExpandedInteger(N->getOperand(1), In1, In2);
EVT HalfVT = In1.getValueType();

SDLoc DL(N);
unsigned Opc = N->getOpcode();
SDValue ShAmt = N->getOperand(2);
EVT ShAmtVT = ShAmt.getValueType();
EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);

// If the shift amount is at least half the bitwidth, swap the inputs.
unsigned HalfVTBits = HalfVT.getScalarSizeInBits();
SDValue AndNode = DAG.getNode(ISD::AND, DL, ShAmtVT, ShAmt,
                              DAG.getConstant(HalfVTBits, DL, ShAmtVT));
SDValue Cond =
    DAG.getSetCC(DL, ShAmtCCVT, AndNode, DAG.getConstant(0, DL, ShAmtVT),
                 Opc == ISD::FSHL ? ISD::SETNE : ISD::SETEQ);

// Expand to a pair of funnel shifts.
EVT NewShAmtVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
SDValue NewShAmt = DAG.getAnyExtOrTrunc(ShAmt, DL, NewShAmtVT);

SDValue Select1 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In1, In2);
SDValue Select2 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In2, In3);
SDValue Select3 = DAG.getNode(ISD::SELECT, DL, HalfVT, Cond, In3, In4);
Lo = DAG.getNode(Opc, DL, HalfVT, Select2, Select1, NewShAmt);
Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
4522}

4524void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
                                         SDValue &Hi) {
EVT VT = N->getValueType(0);
EVT HalfVT =
    EVT::getIntegerVT(*DAG.getContext(), N->getValueSizeInBits(0) / 2);
SDLoc dl(N);

// We assume VSCALE(1) fits into a legal integer.
APInt One(HalfVT.getSizeInBits(), 1);
SDValue VScaleBase = DAG.getVScale(dl, HalfVT, One);
VScaleBase = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, VScaleBase);
SDValue Res = DAG.getNode(ISD::MUL, dl, VT, VScaleBase, N->getOperand(0));
SplitInteger(Res, Lo, Hi);
4537}

4539//===----------------------------------------------------------------------===//
4540//  Integer Operand Expansion
4541//===----------------------------------------------------------------------===//

4543/// ExpandIntegerOperand - This method is called when the specified operand of
4544/// the specified node is found to need expansion.  At this point, all of the
4545/// result types of the node are known to be legal, but other operands of the
4546/// node may need promotion or expansion as well as the specified one.
4547bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
)
           dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalize-types")) { dbgs() << "Expand integer operand: "
; N->dump(&DAG); dbgs() << "\n"; } } while (false
);
SDValue Res = SDValue();

if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
  return false;

switch (N->getOpcode()) {
default:
#ifndef NDEBUG
  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
  N->dump(&DAG); dbgs() << "\n";
#endif
  report_fatal_error("Do not know how to expand this operator's operand!");

case ISD::BITCAST:           Res = ExpandOp_BITCAST(N); break;
case ISD::BR_CC:             Res = ExpandIntOp_BR_CC(N); break;
case ISD::BUILD_VECTOR:      Res = ExpandOp_BUILD_VECTOR(N); break;
case ISD::EXTRACT_ELEMENT:   Res = ExpandOp_EXTRACT_ELEMENT(N); break;
case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
case ISD::SCALAR_TO_VECTOR:  Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
case ISD::SPLAT_VECTOR:      Res = ExpandIntOp_SPLAT_VECTOR(N); break;
case ISD::SELECT_CC:         Res = ExpandIntOp_SELECT_CC(N); break;
case ISD::SETCC:             Res = ExpandIntOp_SETCC(N); break;
case ISD::SETCCCARRY:        Res = ExpandIntOp_SETCCCARRY(N); break;
case ISD::STRICT_SINT_TO_FP:
case ISD::SINT_TO_FP:        Res = ExpandIntOp_SINT_TO_FP(N); break;
case ISD::STORE:   Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
case ISD::TRUNCATE:          Res = ExpandIntOp_TRUNCATE(N); break;
case ISD::STRICT_UINT_TO_FP:
case ISD::UINT_TO_FP:        Res = ExpandIntOp_UINT_TO_FP(N); break;

case ISD::SHL:
case ISD::SRA:
case ISD::SRL:
case ISD::ROTL:
case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
case ISD::RETURNADDR:
case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;

case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
}

// If the result is null, the sub-method took care of registering results etc.
if (!Res.getNode()) return false;

// If the result is N, the sub-method updated N in place.  Tell the legalizer
// core about this.
if (Res.getNode() == N)
  return true;

assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == 1 && "Invalid operand expansion"
) ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && \"Invalid operand expansion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4600
, __extension__ __PRETTY_FUNCTION__))
       "Invalid operand expansion")(static_cast <bool> (Res.getValueType() == N->getValueType
(0) && N->getNumValues() == 1 && "Invalid operand expansion"
) ? void (0) : __assert_fail ("Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && \"Invalid operand expansion\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4600
, __extension__ __PRETTY_FUNCTION__));

ReplaceValueWith(SDValue(N, 0), Res);
return false;
4604}

4606/// IntegerExpandSetCCOperands - Expand the operands of a comparison.  This code
4607/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
4608void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
                                                SDValue &NewRHS,
                                                ISD::CondCode &CCCode,
                                                const SDLoc &dl) {
SDValue LHSLo, LHSHi, RHSLo, RHSHi;
GetExpandedInteger(NewLHS, LHSLo, LHSHi);
GetExpandedInteger(NewRHS, RHSLo, RHSHi);

if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
  if (RHSLo == RHSHi) {
    if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
      if (RHSCST->isAllOnes()) {
        // Equality comparison to -1.
        NewLHS = DAG.getNode(ISD::AND, dl,
                             LHSLo.getValueType(), LHSLo, LHSHi);
        NewRHS = RHSLo;
        return;
      }
    }
  }

  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
  return;
}

// If this is a comparison of the sign bit, just look at the top part.
// X > -1,  x < 0
if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
  if ((CCCode == ISD::SETLT && CST->isZero()) ||    // X < 0
      (CCCode == ISD::SETGT && CST->isAllOnes())) { // X > -1
    NewLHS = LHSHi;
    NewRHS = RHSHi;
    return;
  }

// FIXME: This generated code sucks.
ISD::CondCode LowCC;
switch (CCCode) {
default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp"
, 4649);
case ISD::SETLT:
case ISD::SETULT: LowCC = ISD::SETULT; break;
case ISD::SETGT:
case ISD::SETUGT: LowCC = ISD::SETUGT; break;
case ISD::SETLE:
case ISD::SETULE: LowCC = ISD::SETULE; break;
case ISD::SETGE:
case ISD::SETUGE: LowCC = ISD::SETUGE; break;
}

// LoCmp = lo(op1) < lo(op2)   // Always unsigned comparison
// HiCmp = hi(op1) < hi(op2)   // Signedness depends on operands
// dest  = hi(op1) == hi(op2) ? LoCmp : HiCmp;

// NOTE: on targets without efficient SELECT of bools, we can always use
// this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
                                               nullptr);
SDValue LoCmp, HiCmp;
if (TLI.isTypeLegal(LHSLo.getValueType()) &&
    TLI.isTypeLegal(RHSLo.getValueType()))
  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
                            RHSLo, LowCC, false, DagCombineInfo, dl);
if (!LoCmp.getNode())
  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
                       RHSLo, LowCC);
if (TLI.isTypeLegal(LHSHi.getValueType()) &&
    TLI.isTypeLegal(RHSHi.getValueType()))
  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
                            RHSHi, CCCode, false, DagCombineInfo, dl);
if (!HiCmp.getNode())
  HiCmp =
      DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
                  LHSHi, RHSHi, DAG.getCondCode(CCCode));

ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());

bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
                  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);

// FIXME: Is the HiCmpC->isOne() here correct for
// ZeroOrNegativeOneBooleanContent.
if ((EqAllowed && (HiCmpC && HiCmpC->isZero())) ||
    (!EqAllowed &&
     ((HiCmpC && HiCmpC->isOne()) || (LoCmpC && LoCmpC->isZero())))) {
  // For LE / GE, if high part is known false, ignore the low part.
  // For LT / GT: if low part is known false, return the high part.
  //              if high part is known true, ignore the low part.
  NewLHS = HiCmp;
  NewRHS = SDValue();
  return;
}

if (LHSHi == RHSHi) {
  // Comparing the low bits is enough.
  NewLHS = LoCmp;
  NewRHS = SDValue();
  return;
}

// Lower with SETCCCARRY if the target supports it.
EVT HiVT = LHSHi.getValueType();
EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);

// FIXME: Make all targets support this, then remove the other lowering.
if (HasSETCCCARRY) {
  // SETCCCARRY can detect < and >= directly. For > and <=, flip
  // operands and condition code.
  bool FlipOperands = false;
  switch (CCCode) {
  case ISD::SETGT:  CCCode = ISD::SETLT;  FlipOperands = true; break;
  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
  case ISD::SETLE:  CCCode = ISD::SETGE;  FlipOperands = true; break;
  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
  default: break;
  }
  if (FlipOperands) {
    std::swap(LHSLo, RHSLo);
    std::swap(LHSHi, RHSHi);
  }
  // Perform a wide subtraction, feeding the carry from the low part into
  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
  // zero or positive iff LHS >= RHS.
  EVT LoVT = LHSLo.getValueType();
  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
                            LHSHi, RHSHi, LowCmp.getValue(1),
                            DAG.getCondCode(CCCode));
  NewLHS = Res;
  NewRHS = SDValue();
  return;
}

NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
                           false, DagCombineInfo, dl);
if (!NewLHS.getNode())
  NewLHS =
      DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
NewRHS = SDValue();
4754}

4756SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));

// If ExpandSetCCOperands returned a scalar, we need to compare the result
// against zero to select between true and false values.
if (!NewRHS.getNode()) {
  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
  CCCode = ISD::SETNE;
}

// Update N to have the operands specified.
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
                              DAG.getCondCode(CCCode), NewLHS, NewRHS,
                              N->getOperand(4)), 0);
4772}

4774SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));

// If ExpandSetCCOperands returned a scalar, we need to compare the result
// against zero to select between true and false values.
if (!NewRHS.getNode()) {
  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
  CCCode = ISD::SETNE;
}

// Update N to have the operands specified.
return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
                              N->getOperand(2), N->getOperand(3),
                              DAG.getCondCode(CCCode)), 0);
4790}

4792SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));

// If ExpandSetCCOperands returned a scalar, use it.
if (!NewRHS.getNode()) {
  assert(NewLHS.getValueType() == N->getValueType(0) &&(static_cast <bool> (NewLHS.getValueType() == N->getValueType
(0) && "Unexpected setcc expansion!") ? void (0) : __assert_fail
 ("NewLHS.getValueType() == N->getValueType(0) && \"Unexpected setcc expansion!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4800
, __extension__ __PRETTY_FUNCTION__))
         "Unexpected setcc expansion!")(static_cast <bool> (NewLHS.getValueType() == N->getValueType
(0) && "Unexpected setcc expansion!") ? void (0) : __assert_fail
 ("NewLHS.getValueType() == N->getValueType(0) && \"Unexpected setcc expansion!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4800
, __extension__ __PRETTY_FUNCTION__));
  return NewLHS;
}

// Otherwise, update N to have the operands specified.
return SDValue(
    DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
4807}

4809SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
SDValue LHS = N->getOperand(0);
SDValue RHS = N->getOperand(1);
SDValue Carry = N->getOperand(2);
SDValue Cond = N->getOperand(3);
SDLoc dl = SDLoc(N);

SDValue LHSLo, LHSHi, RHSLo, RHSHi;
GetExpandedInteger(LHS, LHSLo, LHSHi);
GetExpandedInteger(RHS, RHSLo, RHSHi);

// Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
                   LowCmp.getValue(1), Cond);
4825}

4827SDValue DAGTypeLegalizer::ExpandIntOp_SPLAT_VECTOR(SDNode *N) {
// Split the operand and replace with SPLAT_VECTOR_PARTS.
SDValue Lo, Hi;
GetExpandedInteger(N->getOperand(0), Lo, Hi);
return DAG.getNode(ISD::SPLAT_VECTOR_PARTS, SDLoc(N), N->getValueType(0), Lo,
                   Hi);
4833}

4835SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
// The value being shifted is legal, but the shift amount is too big.
// It follows that either the result of the shift is undefined, or the
// upper half of the shift amount is zero.  Just use the lower half.
SDValue Lo, Hi;
GetExpandedInteger(N->getOperand(1), Lo, Hi);
return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
4842}

4844SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
// The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant.  This
// surely makes pretty nice problems on 8/16 bit targets. Just truncate this
// constant to valid type.
SDValue Lo, Hi;
GetExpandedInteger(N->getOperand(0), Lo, Hi);
return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
4851}

4853SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
bool IsStrict = N->isStrictFPOpcode();
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
SDValue Op = N->getOperand(IsStrict ? 1 : 0);
EVT DstVT = N->getValueType(0);
RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Don't know how to expand this SINT_TO_FP!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this SINT_TO_FP!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4860
, __extension__ __PRETTY_FUNCTION__))
       "Don't know how to expand this SINT_TO_FP!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Don't know how to expand this SINT_TO_FP!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this SINT_TO_FP!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4860
, __extension__ __PRETTY_FUNCTION__));
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
std::pair<SDValue, SDValue> Tmp =
    TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);

if (!IsStrict)
  return Tmp.first;

ReplaceValueWith(SDValue(N, 1), Tmp.second);
ReplaceValueWith(SDValue(N, 0), Tmp.first);
return SDValue();
4872}

4874SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
if (N->isAtomic()) {
  // It's typical to have larger CAS than atomic store instructions.
  SDLoc dl(N);
  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
                               N->getMemoryVT(),
                               N->getOperand(0), N->getOperand(2),
                               N->getOperand(1),
                               N->getMemOperand());
  return Swap.getValue(1);
}
if (ISD::isNormalStore(N))
  return ExpandOp_NormalStore(N, OpNo);

assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!")(static_cast <bool> (ISD::isUNINDEXEDStore(N) &&
 "Indexed store during type legalization!") ? void (0) : __assert_fail
 ("ISD::isUNINDEXEDStore(N) && \"Indexed store during type legalization!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4888
, __extension__ __PRETTY_FUNCTION__));
assert(OpNo == 1 && "Can only expand the stored value so far")(static_cast <bool> (OpNo == 1 && "Can only expand the stored value so far"
) ? void (0) : __assert_fail ("OpNo == 1 && \"Can only expand the stored value so far\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4889
, __extension__ __PRETTY_FUNCTION__));

EVT VT = N->getOperand(1).getValueType();
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
SDValue Ch  = N->getChain();
SDValue Ptr = N->getBasePtr();
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
AAMDNodes AAInfo = N->getAAInfo();
SDLoc dl(N);
SDValue Lo, Hi;

assert(NVT.isByteSized() && "Expanded type not byte sized!")(static_cast <bool> (NVT.isByteSized() && "Expanded type not byte sized!"
) ? void (0) : __assert_fail ("NVT.isByteSized() && \"Expanded type not byte sized!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4900
, __extension__ __PRETTY_FUNCTION__));

if (N->getMemoryVT().bitsLE(NVT)) {
  GetExpandedInteger(N->getValue(), Lo, Hi);
  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
                           N->getMemoryVT(), N->getOriginalAlign(), MMOFlags,
                           AAInfo);
}

if (DAG.getDataLayout().isLittleEndian()) {
  // Little-endian - low bits are at low addresses.
  GetExpandedInteger(N->getValue(), Lo, Hi);

  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
                    N->getOriginalAlign(), MMOFlags, AAInfo);

  unsigned ExcessBits =
    N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);

  // Increment the pointer to the other half.
  unsigned IncrementSize = NVT.getSizeInBits()/8;
  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
                         N->getPointerInfo().getWithOffset(IncrementSize),
                         NEVT, N->getOriginalAlign(), MMOFlags, AAInfo);
  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
}

// Big-endian - high bits are at low addresses.  Favor aligned stores at
// the cost of some bit-fiddling.
GetExpandedInteger(N->getValue(), Lo, Hi);

EVT ExtVT = N->getMemoryVT();
unsigned EBytes = ExtVT.getStoreSize();
unsigned IncrementSize = NVT.getSizeInBits()/8;
unsigned ExcessBits = (EBytes - IncrementSize)*8;
EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
                             ExtVT.getSizeInBits() - ExcessBits);

if (ExcessBits < NVT.getSizeInBits()) {
  // Transfer high bits from the top of Lo to the bottom of Hi.
  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
                   DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
                                   TLI.getPointerTy(DAG.getDataLayout())));
  Hi = DAG.getNode(
      ISD::OR, dl, NVT, Hi,
      DAG.getNode(ISD::SRL, dl, NVT, Lo,
                  DAG.getConstant(ExcessBits, dl,
                                  TLI.getPointerTy(DAG.getDataLayout()))));
}

// Store both the high bits and maybe some of the low bits.
Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT,
                       N->getOriginalAlign(), MMOFlags, AAInfo);

// Increment the pointer to the other half.
Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
// Store the lowest ExcessBits bits in the second half.
Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
                       N->getPointerInfo().getWithOffset(IncrementSize),
                       EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
                       N->getOriginalAlign(), MMOFlags, AAInfo);
return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
4964}

4966SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
SDValue InL, InH;
GetExpandedInteger(N->getOperand(0), InL, InH);
// Just truncate the low part of the source.
return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
4971}

4973SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
bool IsStrict = N->isStrictFPOpcode();
SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
SDValue Op = N->getOperand(IsStrict ? 1 : 0);
EVT DstVT = N->getValueType(0);
RTLIB::Libcall LC = RTLIB::getUINTTOFP(Op.getValueType(), DstVT);
assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Don't know how to expand this UINT_TO_FP!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this UINT_TO_FP!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4980
, __extension__ __PRETTY_FUNCTION__))
       "Don't know how to expand this UINT_TO_FP!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Don't know how to expand this UINT_TO_FP!") ? void (0) : __assert_fail
 ("LC != RTLIB::UNKNOWN_LIBCALL && \"Don't know how to expand this UINT_TO_FP!\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 4980
, __extension__ __PRETTY_FUNCTION__));
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setSExt(true);
std::pair<SDValue, SDValue> Tmp =
    TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N), Chain);

if (!IsStrict)
  return Tmp.first;

ReplaceValueWith(SDValue(N, 1), Tmp.second);
ReplaceValueWith(SDValue(N, 0), Tmp.first);
return SDValue();
4992}

4994SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
SDLoc dl(N);
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
                             cast<AtomicSDNode>(N)->getMemoryVT(),
                             N->getOperand(0),
                             N->getOperand(1), N->getOperand(2),
                             cast<AtomicSDNode>(N)->getMemOperand());
return Swap.getValue(1);
5002}

5004SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(SDNode *N) {
SDLoc dl(N);

SDValue V0 = GetPromotedInteger(N->getOperand(0));
SDValue V1 = GetPromotedInteger(N->getOperand(1));
EVT OutVT = V0.getValueType();

return DAG.getNode(ISD::VECTOR_SPLICE, dl, OutVT, V0, V1, N->getOperand(2));
5012}

5014SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {

EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5018
, __extension__ __PRETTY_FUNCTION__));
EVT NOutVTElem = NOutVT.getVectorElementType();

SDLoc dl(N);
SDValue BaseIdx = N->getOperand(1);

// TODO: We may be able to use this for types other than scalable
// vectors and fix those tests that expect BUILD_VECTOR to be used
if (OutVT.isScalableVector()) {
  SDValue InOp0 = N->getOperand(0);
  EVT InVT = InOp0.getValueType();

  // Try and extract from a smaller type so that it eventually falls
  // into the promotion code below.
  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector ||
      getTypeAction(InVT) == TargetLowering::TypeLegal) {
    EVT NInVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
    unsigned NElts = NInVT.getVectorMinNumElements();
    uint64_t IdxVal = cast<ConstantSDNode>(BaseIdx)->getZExtValue();

    SDValue Step1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NInVT, InOp0,
                                DAG.getConstant(alignDown(IdxVal, NElts), dl,
                                                BaseIdx.getValueType()));
    SDValue Step2 = DAG.getNode(
        ISD::EXTRACT_SUBVECTOR, dl, OutVT, Step1,
        DAG.getConstant(IdxVal % NElts, dl, BaseIdx.getValueType()));
    return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Step2);
  }

  // Try and extract from a widened type.
  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
    SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
    SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), OutVT, Ops);
    return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
  }

  // Promote operands and see if this is handled by target lowering,
  // Otherwise, use the BUILD_VECTOR approach below
  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
    // Collect the (promoted) operands
    SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };

    EVT PromEltVT = Ops[0].getValueType().getVectorElementType();
    assert(PromEltVT.bitsLE(NOutVTElem) &&(static_cast <bool> (PromEltVT.bitsLE(NOutVTElem) &&
 "Promoted operand has an element type greater than result") ?
 void (0) : __assert_fail ("PromEltVT.bitsLE(NOutVTElem) && \"Promoted operand has an element type greater than result\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5062
, __extension__ __PRETTY_FUNCTION__))
           "Promoted operand has an element type greater than result")(static_cast <bool> (PromEltVT.bitsLE(NOutVTElem) &&
 "Promoted operand has an element type greater than result") ?
 void (0) : __assert_fail ("PromEltVT.bitsLE(NOutVTElem) && \"Promoted operand has an element type greater than result\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5062
, __extension__ __PRETTY_FUNCTION__));

    EVT ExtVT = NOutVT.changeVectorElementType(PromEltVT);
    SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), ExtVT, Ops);
    return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, Ext);
  }
}

if (OutVT.isScalableVector())
  report_fatal_error("Unable to promote scalable types using BUILD_VECTOR");

SDValue InOp0 = N->getOperand(0);
if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
  InOp0 = GetPromotedInteger(N->getOperand(0));

EVT InVT = InOp0.getValueType();

unsigned OutNumElems = OutVT.getVectorNumElements();
SmallVector<SDValue, 8> Ops;
Ops.reserve(OutNumElems);
for (unsigned i = 0; i != OutNumElems; ++i) {

  // Extract the element from the original vector.
  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
    BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
    InVT.getVectorElementType(), N->getOperand(0), Index);

  SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
  // Insert the converted element to the new vector.
  Ops.push_back(Op);
}

return DAG.getBuildVector(NOutVT, dl, Ops);
5096}

5098SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(SDNode *N) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5101
, __extension__ __PRETTY_FUNCTION__));

SDLoc dl(N);
SDValue Vec = N->getOperand(0);
SDValue SubVec = N->getOperand(1);
SDValue Idx = N->getOperand(2);

EVT SubVecVT = SubVec.getValueType();
EVT NSubVT =
    EVT::getVectorVT(*DAG.getContext(), NOutVT.getVectorElementType(),
                     SubVecVT.getVectorElementCount());

Vec = GetPromotedInteger(Vec);
SubVec = DAG.getNode(ISD::ANY_EXTEND, dl, NSubVT, SubVec);

return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NOutVT, Vec, SubVec, Idx);
5117}

5119SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(SDNode *N) {
SDLoc dl(N);

SDValue V0 = GetPromotedInteger(N->getOperand(0));
EVT OutVT = V0.getValueType();

return DAG.getNode(ISD::VECTOR_REVERSE, dl, OutVT, V0);
5126}

5128SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
EVT VT = N->getValueType(0);
SDLoc dl(N);

ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());

SDValue V0 = GetPromotedInteger(N->getOperand(0));
SDValue V1 = GetPromotedInteger(N->getOperand(1));
EVT OutVT = V0.getValueType();

return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
5140}


5143SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5146
, __extension__ __PRETTY_FUNCTION__));
unsigned NumElems = N->getNumOperands();
EVT NOutVTElem = NOutVT.getVectorElementType();

SDLoc dl(N);

SmallVector<SDValue, 8> Ops;
Ops.reserve(NumElems);
for (unsigned i = 0; i != NumElems; ++i) {
  SDValue Op;
  // BUILD_VECTOR integer operand types are allowed to be larger than the
  // result's element type. This may still be true after the promotion. For
  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
    Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
  else
    Op = N->getOperand(i);
  Ops.push_back(Op);
}

return DAG.getBuildVector(NOutVT, dl, Ops);
5168}

5170SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {

SDLoc dl(N);

assert(!N->getOperand(0).getValueType().isVector() &&(static_cast <bool> (!N->getOperand(0).getValueType(
).isVector() && "Input must be a scalar") ? void (0) :
 __assert_fail ("!N->getOperand(0).getValueType().isVector() && \"Input must be a scalar\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5175
, __extension__ __PRETTY_FUNCTION__))
       "Input must be a scalar")(static_cast <bool> (!N->getOperand(0).getValueType(
).isVector() && "Input must be a scalar") ? void (0) :
 __assert_fail ("!N->getOperand(0).getValueType().isVector() && \"Input must be a scalar\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5175
, __extension__ __PRETTY_FUNCTION__));

EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5179
, __extension__ __PRETTY_FUNCTION__));
EVT NOutVTElem = NOutVT.getVectorElementType();

SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));

return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
5185}

5187SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
SDLoc dl(N);

SDValue SplatVal = N->getOperand(0);

assert(!SplatVal.getValueType().isVector() && "Input must be a scalar")(static_cast <bool> (!SplatVal.getValueType().isVector(
) && "Input must be a scalar") ? void (0) : __assert_fail
 ("!SplatVal.getValueType().isVector() && \"Input must be a scalar\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5192
, __extension__ __PRETTY_FUNCTION__));

EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "Type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "Type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"Type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5196
, __extension__ __PRETTY_FUNCTION__));
EVT NOutElemVT = NOutVT.getVectorElementType();

SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);

return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
5202}

5204SDValue DAGTypeLegalizer::PromoteIntRes_STEP_VECTOR(SDNode *N) {
SDLoc dl(N);
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "Type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "Type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"Type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5208
, __extension__ __PRETTY_FUNCTION__));
APInt StepVal = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
return DAG.getStepVector(dl, NOutVT,
                         StepVal.sext(NOutVT.getScalarSizeInBits()));
5212}

5214SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
SDLoc dl(N);

EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5219
, __extension__ __PRETTY_FUNCTION__));

unsigned NumOperands = N->getNumOperands();
unsigned NumOutElem = NOutVT.getVectorMinNumElements();
EVT OutElemTy = NOutVT.getVectorElementType();
if (OutVT.isScalableVector()) {
  // Find the largest promoted element type for each of the operands.
  SDUse *MaxSizedValue = std::max_element(
      N->op_begin(), N->op_end(), [](const SDValue &A, const SDValue &B) {
        EVT AVT = A.getValueType().getVectorElementType();
        EVT BVT = B.getValueType().getVectorElementType();
        return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
      });
  EVT MaxElementVT = MaxSizedValue->getValueType().getVectorElementType();

  // Then promote all vectors to the largest element type.
  SmallVector<SDValue, 8> Ops;
  for (unsigned I = 0; I < NumOperands; ++I) {
    SDValue Op = N->getOperand(I);
    EVT OpVT = Op.getValueType();
    if (getTypeAction(OpVT) == TargetLowering::TypePromoteInteger)
      Op = GetPromotedInteger(Op);
    else
      assert(getTypeAction(OpVT) == TargetLowering::TypeLegal &&(static_cast <bool> (getTypeAction(OpVT) == TargetLowering
::TypeLegal && "Unhandled legalization type") ? void (
0) : __assert_fail ("getTypeAction(OpVT) == TargetLowering::TypeLegal && \"Unhandled legalization type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5243
, __extension__ __PRETTY_FUNCTION__))
             "Unhandled legalization type")(static_cast <bool> (getTypeAction(OpVT) == TargetLowering
::TypeLegal && "Unhandled legalization type") ? void (
0) : __assert_fail ("getTypeAction(OpVT) == TargetLowering::TypeLegal && \"Unhandled legalization type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5243
, __extension__ __PRETTY_FUNCTION__));

    if (OpVT.getVectorElementType().getScalarSizeInBits() <
        MaxElementVT.getScalarSizeInBits())
      Op = DAG.getAnyExtOrTrunc(Op, dl,
                                OpVT.changeVectorElementType(MaxElementVT));
    Ops.push_back(Op);
  }

  // Do the CONCAT on the promoted type and finally truncate to (the promoted)
  // NOutVT.
  return DAG.getAnyExtOrTrunc(
      DAG.getNode(ISD::CONCAT_VECTORS, dl,
                  OutVT.changeVectorElementType(MaxElementVT), Ops),
      dl, NOutVT);
}

unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
assert(NumElem * NumOperands == NumOutElem &&(static_cast <bool> (NumElem * NumOperands == NumOutElem
 && "Unexpected number of elements") ? void (0) : __assert_fail
 ("NumElem * NumOperands == NumOutElem && \"Unexpected number of elements\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5262
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected number of elements")(static_cast <bool> (NumElem * NumOperands == NumOutElem
 && "Unexpected number of elements") ? void (0) : __assert_fail
 ("NumElem * NumOperands == NumOutElem && \"Unexpected number of elements\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5262
, __extension__ __PRETTY_FUNCTION__));

// Take the elements from the first vector.
SmallVector<SDValue, 8> Ops(NumOutElem);
for (unsigned i = 0; i < NumOperands; ++i) {
  SDValue Op = N->getOperand(i);
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
    Op = GetPromotedInteger(Op);
  EVT SclrTy = Op.getValueType().getVectorElementType();
  assert(NumElem == Op.getValueType().getVectorNumElements() &&(static_cast <bool> (NumElem == Op.getValueType().getVectorNumElements
() && "Unexpected number of elements") ? void (0) : __assert_fail
 ("NumElem == Op.getValueType().getVectorNumElements() && \"Unexpected number of elements\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5272
, __extension__ __PRETTY_FUNCTION__))
         "Unexpected number of elements")(static_cast <bool> (NumElem == Op.getValueType().getVectorNumElements
() && "Unexpected number of elements") ? void (0) : __assert_fail
 ("NumElem == Op.getValueType().getVectorNumElements() && \"Unexpected number of elements\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5272
, __extension__ __PRETTY_FUNCTION__));

  for (unsigned j = 0; j < NumElem; ++j) {
    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
                              DAG.getVectorIdxConstant(j, dl));
    Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
  }
}

return DAG.getBuildVector(NOutVT, dl, Ops);
5282}

5284SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
assert(NVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5287
, __extension__ __PRETTY_FUNCTION__));

SDLoc dl(N);

// For operands whose TypeAction is to promote, extend the promoted node
// appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
// type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
// type..
if (getTypeAction(N->getOperand(0).getValueType())
    == TargetLowering::TypePromoteInteger) {
  SDValue Promoted;

  switch(N->getOpcode()) {
    case ISD::SIGN_EXTEND_VECTOR_INREG:
      Promoted = SExtPromotedInteger(N->getOperand(0));
      break;
    case ISD::ZERO_EXTEND_VECTOR_INREG:
      Promoted = ZExtPromotedInteger(N->getOperand(0));
      break;
    case ISD::ANY_EXTEND_VECTOR_INREG:
      Promoted = GetPromotedInteger(N->getOperand(0));
      break;
    default:
      llvm_unreachable("Node has unexpected Opcode")::llvm::llvm_unreachable_internal("Node has unexpected Opcode"
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5310
);
  }
  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
}

// Directly extend to the appropriate transform-to type.
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
5317}

5319SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
assert(NOutVT.isVector() && "This type must be promoted to a vector type")(static_cast <bool> (NOutVT.isVector() && "This type must be promoted to a vector type"
) ? void (0) : __assert_fail ("NOutVT.isVector() && \"This type must be promoted to a vector type\""
, "llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp", 5322
, __extension__ __PRETTY_FUNCTION__));

EVT NOutVTElem = NOutVT.getVectorElementType();

SDLoc dl(N);
SDValue V0 = GetPromotedInteger(N->getOperand(0));

SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
  NOutVTElem, N->getOperand(1));
return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
  V0, ConvElem, N->getOperand(2));
5333}

5335SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
// The VECREDUCE result size may be larger than the element size, so
// we can simply change the result type.
SDLoc dl(N);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
return DAG.getNode(N->getOpcode(), dl, NVT, N->ops());
5341}

5343SDValue DAGTypeLegalizer::PromoteIntRes_VP_REDUCE(SDNode *N) {
// The VP_REDUCE result size may be larger than the element size, so we can
// simply change the result type. However the start value and result must be
// the same.
SDLoc DL(N);
SDValue Start = PromoteIntOpVectorReduction(N, N->getOperand(0));
return DAG.getNode(N->getOpcode(), DL, Start.getValueType(), Start,
                   N->getOperand(1), N->getOperand(2), N->getOperand(3));
5351}

5353SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
SDLoc dl(N);
SDValue V0 = GetPromotedInteger(N->getOperand(0));
SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
                                TLI.getVectorIdxTy(DAG.getDataLayout()));
SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
  V0->getValueType(0).getScalarType(), V0, V1);

// EXTRACT_VECTOR_ELT can return types which are wider than the incoming
// element types. If this is the case then we need to expand the outgoing
// value and not truncate it.
return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5365}

5367SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(SDNode *N) {
SDLoc dl(N);
// The result type is equal to the first input operand's type, so the
// type that needs promoting must be the second source vector.
SDValue V0 = N->getOperand(0);
SDValue V1 = GetPromotedInteger(N->getOperand(1));
SDValue Idx = N->getOperand(2);
EVT PromVT = EVT::getVectorVT(*DAG.getContext(),
                              V1.getValueType().getVectorElementType(),
                              V0.getValueType().getVectorElementCount());
V0 = DAG.getAnyExtOrTrunc(V0, dl, PromVT);
SDValue Ext = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, PromVT, V0, V1, Idx);
return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
5380}

5382SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
SDLoc dl(N);
SDValue V0 = GetPromotedInteger(N->getOperand(0));
MVT InVT = V0.getValueType().getSimpleVT();
MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
                             N->getValueType(0).getVectorNumElements());
SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
5390}

5392SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
SDLoc dl(N);

EVT ResVT = N->getValueType(0);
unsigned NumElems = N->getNumOperands();

if (ResVT.isScalableVector()) {
  SDValue ResVec = DAG.getUNDEF(ResVT);

  for (unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
    SDValue Op = N->getOperand(OpIdx);
    unsigned OpNumElts = Op.getValueType().getVectorMinNumElements();
    ResVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ResVec, Op,
                         DAG.getIntPtrConstant(OpIdx * OpNumElts, dl));
  }

  return ResVec;
}

EVT RetSclrTy = N->getValueType(0).getVectorElementType();

SmallVector<SDValue, 8> NewOps;
NewOps.reserve(NumElems);

// For each incoming vector
for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();

  for (unsigned i=0; i<NumElem; ++i) {
    // Extract element from incoming vector
    SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
                             DAG.getVectorIdxConstant(i, dl));
    SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
    NewOps.push_back(Tr);
  }
}

return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
5432}

←

/build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>

57namespace llvm {

59class APInt;
60class Constant;
61class GlobalValue;
62class MachineBasicBlock;
63class MachineConstantPoolValue;
64class MCSymbol;
65class raw_ostream;
66class SDNode;
67class SelectionDAG;
68class Type;
69class Value;

71void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

74/// This represents a list of ValueType's that has been intern'd by
75/// a SelectionDAG.  Instances of this simple value class are returned by
76/// SelectionDAG::getVTList(...).
77///
78struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
81};

83namespace ISD {

/// Node predicates

87/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
88/// same constant or undefined, return true and return the constant value in
89/// \p SplatValue.
90bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

92/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
93/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
94/// true, it only checks BUILD_VECTOR.
95bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

98/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
99/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
100/// only checks BUILD_VECTOR.
101bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

104/// Return true if the specified node is a BUILD_VECTOR where all of the
105/// elements are ~0 or undef.
106bool isBuildVectorAllOnes(const SDNode *N);

108/// Return true if the specified node is a BUILD_VECTOR where all of the
109/// elements are 0 or undef.
110bool isBuildVectorAllZeros(const SDNode *N);

112/// Return true if the specified node is a BUILD_VECTOR node of all
113/// ConstantSDNode or undef.
114bool isBuildVectorOfConstantSDNodes(const SDNode *N);

116/// Return true if the specified node is a BUILD_VECTOR node of all
117/// ConstantFPSDNode or undef.
118bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

120/// Return true if the node has at least one operand and all operands of the
121/// specified node are ISD::UNDEF.
122bool allOperandsUndef(const SDNode *N);

124} // end namespace ISD

126//===----------------------------------------------------------------------===//
127/// Unlike LLVM values, Selection DAG nodes may return multiple
128/// values as the result of a computation.  Many nodes return multiple values,
129/// from loads (which define a token and a return value) to ADDC (which returns
130/// a result and a carry value), to calls (which may return an arbitrary number
131/// of values).
132///
133/// As such, each use of a SelectionDAG computation must indicate the node that
134/// computes it as well as which return value to use from that node.  This pair
135/// of information is represented with the SDValue value type.
136///
137class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
unsigned ResNo = 0;     // Which return value of the node we are using.

143public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
229};

231template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
252};

254/// Allow casting operators to work directly on
255/// SDValues as if they were SDNode*'s.
256template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
}
262};
263template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
269};

271/// Represents a use of a SDNode. This class holds an SDValue,
272/// which records the SDNode being used and the result number, a
273/// pointer to the SDNode using the value, and Next and Prev pointers,
274/// which link together all the uses of an SDNode.
275///
276class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

286public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

326private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
355};

357/// simplify_type specializations - Allow casting operators to work directly on
358/// SDValues as if they were SDNode*'s.
359template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
365};

367/// These are IR-level optimization flags that may be propagated to SDNodes.
368/// TODO: This data structure should be shared by the IR optimizer and the
369/// the backend.
370struct SDNodeFlags {
371private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

390public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
450};

452/// Represents one node in the SelectionDAG.
453///
454class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
455private:
/// The operation that this node performs.
int16_t NodeType;

459protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with NodeType.

464#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
465// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
466// and give the `pack` pragma push semantics.
467#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
468#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
469#else
470#define BEGIN_TWO_BYTE_PACK()
471#define END_TWO_BYTE_PACK()
472#endif

474BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class VPBaseLoadStoreSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;
  friend class VPGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   VPGatherScatterSDNode => enum ISD::MemIndexType
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class VPLoadSDNode;
  friend class VPStridedLoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;
  friend class VPGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class VPStoreSDNode;
  friend class VPStridedStoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;
  friend class VPScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
566END_TWO_BYTE_PACK()
567#undef BEGIN_TWO_BYTE_PACK
568#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

580private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

617public:
/// Unique and persistent id per SDNode in the DAG. Used for debug printing.
/// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS`
/// intentionally because it adds unneeded complexity without noticeable
/// benefits (see discussion with @thakis in D120714).
uint16_t PersistentId;

//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned short)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
675#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
677#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node is a vector predication operation.
bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 693, __extension__
 __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using iterator_category = std::forward_iterator_tag;
  using value_type = SDUse;
  using difference_type = std::ptrdiff_t;
  using pointer = value_type *;
  using reference = value_type &;

  use_iterator() = default;
  use_iterator(const use_iterator &I) = default;

  bool operator==(const use_iterator &x) const { return Op == x.Op; }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 760, __extension__
 __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 771, __extension__
 __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 781, __extension__
 __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 909, __extension__
 __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 971, __extension__
 __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1058protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
 "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1072, __extension__
 __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1074, __extension__
 __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1074, __extension__
 __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1079};

1081/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1082/// into SDNode creation functions.
1083/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1084/// from the original Instruction, and IROrder is the ordinal position of
1085/// the instruction.
1086/// When an SDNode is created after the DAG is being built, both DebugLoc and
1087/// the IROrder are propagated from the original SDNode.
1088/// So SDLoc class provides two constructors besides the default one, one to
1089/// be used by the DAGBuilder, the other to be used by others.
1090class SDLoc {
1091private:
DebugLoc DL;
int IROrder = 0;

1095public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1100, __extension__
 __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1107};

1109// Define inline functions from the SDValue class.

1111inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1117, __extension__
 __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1117, __extension__
 __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1118, __extension__
 __PRETTY_FUNCTION__));
1119}

1121inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
1123}

1125inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
12
←
Called C++ object pointer is null
1127}

1129inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1131}

1133inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1135}

1137inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1139}

1141inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1143}

1145inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1147}

1149inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1151}

1153inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1155}

1157inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1159}

1161inline bool SDValue::isUndef() const {
return Node->isUndef();
1163}

1165inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1167}

1169inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
1171}

1173inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1175}

1177inline void SDValue::dump() const {
return Node->dump();
1179}

1181inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1183}

1185inline void SDValue::dumpr() const {
return Node->dumpr();
1187}

1189inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1191}

1193// Define inline functions from the SDUse class.

1195inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode())
  V->addUse(*this);
1200}

1202inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V->addUse(*this);
1205}

1207inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1211}

1213/// This class is used to form a handle around another node that
1214/// is persistent and is updated across invocations of replaceAllUsesWith on its
1215/// operand.  This node should be directly created by end-users and not added to
1216/// the AllNodes list.
1217class HandleSDNode : public SDNode {
SDUse Op;

1220public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1239};

1241class AddrSpaceCastSDNode : public SDNode {
1242private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1246public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1256};

1258/// This is an abstract virtual class for memory operations.
1259class MemSDNode : public SDNode {
1260private:
// VT of in-memory value.
EVT MemoryVT;

1264protected:
/// Memory reference information.
MachineMemOperand *MMO;

1268public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }
// FIXME: Remove once transition to getAlign is over.
unsigned getAlignment() const { return MMO->getAlign().value(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getSuccessOrdering() const {
  return MMO->getSuccessOrdering();
}

/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation.  (For operations
/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::VP_STORE:
  case ISD::MSTORE:
  case ISD::VP_SCATTER:
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  switch (N->getOpcode()) {
  case ISD::LOAD:
  case ISD::STORE:
  case ISD::PREFETCH:
  case ISD::ATOMIC_CMP_SWAP:
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  case ISD::ATOMIC_SWAP:
  case ISD::ATOMIC_LOAD_ADD:
  case ISD::ATOMIC_LOAD_SUB:
  case ISD::ATOMIC_LOAD_AND:
  case ISD::ATOMIC_LOAD_CLR:
  case ISD::ATOMIC_LOAD_OR:
  case ISD::ATOMIC_LOAD_XOR:
  case ISD::ATOMIC_LOAD_NAND:
  case ISD::ATOMIC_LOAD_MIN:
  case ISD::ATOMIC_LOAD_MAX:
  case ISD::ATOMIC_LOAD_UMIN:
  case ISD::ATOMIC_LOAD_UMAX:
  case ISD::ATOMIC_LOAD_FADD:
  case ISD::ATOMIC_LOAD_FSUB:
  case ISD::ATOMIC_LOAD:
  case ISD::ATOMIC_STORE:
  case ISD::MLOAD:
  case ISD::MSTORE:
  case ISD::MGATHER:
  case ISD::MSCATTER:
  case ISD::VP_LOAD:
  case ISD::VP_STORE:
  case ISD::VP_GATHER:
  case ISD::VP_SCATTER:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return true;
  default:
    return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
  }
}
1423};

1425/// This is an SDNode representing atomic operations.
1426class AtomicSDNode : public MemSDNode {
1427public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1432, __extension__
 __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1432, __extension__
 __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1449, __extension__
 __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1474};

1476/// This SDNode is used for target intrinsics that touch
1477/// memory and need an associated MachineMemOperand. Its opcode may be
1478/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1479/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1480class MemIntrinsicSDNode : public MemSDNode {
1481public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1496};

1498/// This SDNode is used to implement the code generator
1499/// support for the llvm IR shufflevector instruction.  It combines elements
1500/// from two input vectors into a new input vector, with the selection and
1501/// ordering of elements determined by an array of integers, referred to as
1502/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1503/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1504/// An index of -1 is treated as undef, such that the code generator may put
1505/// any value in the corresponding element of the result.
1506class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1511protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1517public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return makeArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
 ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1524, __extension__
 __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1531, __extension__
 __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1562};

1564class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1576public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isZero() const { return Value->isZero(); }
// NOTE: This is soft-deprecated.  Please use `isZero()` instead.
bool isNullValue() const { return isZero(); }
bool isAllOnes() const { return Value->isMinusOne(); }
// NOTE: This is soft-deprecated.  Please use `isAllOnes()` instead.
bool isAllOnesValue() const { return isAllOnes(); }
bool isMaxSignedValue() const { return Value->isMaxValue(true); }
bool isMinSignedValue() const { return Value->isMinValue(true); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1603};

1605uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1607}

1609const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1611}

1613class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1623public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1658};

1660/// Returns true if \p V is a constant integer zero.
1661bool isNullConstant(SDValue V);

1663/// Returns true if \p V is an FP constant with a value of positive zero.
1664bool isNullFPConstant(SDValue V);

1666/// Returns true if \p V is an integer constant with all bits set.
1667bool isAllOnesConstant(SDValue V);

1669/// Returns true if \p V is a constant integer one.
1670bool isOneConstant(SDValue V);

1672/// Returns true if \p V is a constant min signed integer value.
1673bool isMinSignedConstant(SDValue V);

1675/// Return the non-bitcasted source operand of \p V if it exists.
1676/// If \p V is not a bitcasted value, it is returned as-is.
1677SDValue peekThroughBitcasts(SDValue V);

1679/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1680/// If \p V is not a bitcasted one-use value, it is returned as-is.
1681SDValue peekThroughOneUseBitcasts(SDValue V);

1683/// Return the non-extracted vector source operand of \p V if it exists.
1684/// If \p V is not an extracted subvector, it is returned as-is.
1685SDValue peekThroughExtractSubvectors(SDValue V);

1687/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1688/// constant is canonicalized to be operand 1.
1689bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1691/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1692ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1695/// Returns the SDNode if it is a demanded constant splat BuildVector or
1696/// constant int.
1697ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1701/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1702ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1704/// Returns the SDNode if it is a demanded constant splat BuildVector or
1705/// constant float.
1706ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1709/// Return true if the value is a constant 0 integer or a splatted vector of
1710/// a constant 0 integer (with no undefs by default).
1711/// Build vector implicit truncation is not an issue for null values.
1712bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1714/// Return true if the value is a constant 1 integer or a splatted vector of a
1715/// constant 1 integer (with no undefs).
1716/// Does not permit build vector implicit truncation.
1717bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1719/// Return true if the value is a constant -1 integer or a splatted vector of a
1720/// constant -1 integer (with no undefs).
1721/// Does not permit build vector implicit truncation.
1722bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1724/// Return true if \p V is either a integer or FP constant.
1725inline bool isIntOrFPConstant(SDValue V) {
return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1727}

1729class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1740public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1753};

1755class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1765public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1772};

1774/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1775/// the offet and size that are started/ended in the underlying FrameIndex.
1776class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1784public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1791, __extension__
 __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1795, __extension__
 __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1804};

1806/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1807/// the index of the basic block being probed. A pseudo probe serves as a place
1808/// holder and will be removed at the end of compilation. It does not have any
1809/// operand because we do not want the instruction selection to deal with any.
1810class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1821public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1830};

1832class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1843public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1851};

1853class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1869, __extension__
 __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1878, __extension__
 __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1883public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1889, __extension__
 __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1894, __extension__
 __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1913};

1915/// Completely target-dependent object reference.
1916class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1923public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1935};

1937class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1949public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1955};

1957/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1958class BuildVectorSDNode : public SDNode {
1959public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

/// Extract the raw bit data from a build vector of Undef, Constant or
/// ConstantFP node elements. Each raw bit element will be \p
/// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
/// undefined elements are flagged in \p UndefElements.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                        SmallVectorImpl<APInt> &RawBitElements,
                        BitVector &UndefElements) const;

bool isConstant() const;

/// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements.
/// Undef elements are treated as zero, and entirely undefined elements are
/// flagged in \p DstUndefElements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                          SmallVectorImpl<APInt> &DstBitElements,
                          ArrayRef<APInt> SrcBitElements,
                          BitVector &DstUndefElements,
                          const BitVector &SrcUndefElements);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2085};

2087/// An SDNode that holds an arbitrary LLVM IR Value. This is
2088/// used when the SelectionDAG needs to make a simple reference to something
2089/// in the LLVM IR representation.
2090///
2091class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2100public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2107};

2109class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2118public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2124};

2126class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2134public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2140};

2142class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2152public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2158};

2160class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2172public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2181};

2183class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
 "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2190, __extension__
 __PRETTY_FUNCTION__));
}

2193public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2200};

2202class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2213public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2221};

2223class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2231public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2237};

2239class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2248public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2254};

2256/// This class is used to represent EVT's, which are used
2257/// to parameterize some operations.
2258class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2267public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2273};

2275/// Base class for LoadSDNode and StoreSDNode
2276class LSBaseSDNode : public MemSDNode {
2277public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2283, __extension__
 __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2306};

2308/// This class is used to represent ISD::LOAD nodes.
2309class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2317, __extension__
 __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2318, __extension__
 __PRETTY_FUNCTION__));
}

2321public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2334};

2336/// This class is used to represent ISD::STORE nodes.
2337class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2345, __extension__
 __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2346, __extension__
 __PRETTY_FUNCTION__));
}

2349public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2365};

2367/// This base class is used to represent VP_LOAD, VP_STORE,
2368/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes
2369class VPBaseLoadStoreSDNode : public MemSDNode {
2370public:
friend class SelectionDAG;

VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &DL, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2379, __extension__
 __PRETTY_FUNCTION__));
}

// VPStridedStoreSDNode (Chain, Data, Ptr,    Offset, Stride, Mask, EVL)
// VPStoreSDNode        (Chain, Data, Ptr,    Offset, Mask,   EVL)
// VPStridedLoadSDNode  (Chain, Ptr,  Offset, Stride, Mask,   EVL)
// VPLoadSDNode         (Chain, Ptr,  Offset, Mask,   EVL)
// Mask is a vector of i1 elements;
// the type of EVL is TLI.getVPExplicitVectorLengthTy().
const SDValue &getOffset() const {
  return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
                     getOpcode() == ISD::VP_LOAD)
                        ? 2
                        : 3);
}
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
                     getOpcode() == ISD::VP_LOAD)
                        ? 1
                        : 2);
}
const SDValue &getMask() const {
  switch (getOpcode()) {
  default:
    llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2403);
  case ISD::VP_LOAD:
    return getOperand(3);
  case ISD::VP_STORE:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return getOperand(4);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(5);
  }
}
const SDValue &getVectorLength() const {
  switch (getOpcode()) {
  default:
    llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2416);
  case ISD::VP_LOAD:
    return getOperand(4);
  case ISD::VP_STORE:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return getOperand(5);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(6);
  }
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
         N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE ||
         N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
}
2444};

2446/// This class is used to represent a VP_LOAD node
2447class VPLoadSDNode : public VPBaseLoadStoreSDNode {
2448public:
friend class SelectionDAG;

VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
             ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
             EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = isExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getVectorLength() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2472};

2474/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
2475class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode {
2476public:
friend class SelectionDAG;

VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
                    ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                    bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs,
                            AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getStride() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2502};

2504/// This class is used to represent a VP_STORE node
2505class VPStoreSDNode : public VPBaseLoadStoreSDNode {
2506public:
friend class SelectionDAG;

VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
              ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
              EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_STORE;
}
2537};

2539/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
2540class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode {
2541public:
friend class SelectionDAG;

VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
                     ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing,
                     EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL,
                            VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
  StoreSDNodeBits.IsCompressing = IsCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getStride() const { return getOperand(4); }
const SDValue &getMask() const { return getOperand(5); }
const SDValue &getVectorLength() const { return getOperand(6); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE;
}
2574};

2576/// This base class is used to represent MLOAD and MSTORE nodes
2577class MaskedLoadStoreSDNode : public MemSDNode {
2578public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2587, __extension__
 __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2616};

2618/// This class is used to represent an MLOAD node
2619class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2620public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2645};

2647/// This class is used to represent an MSTORE node
2648class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2649public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2679};

2681/// This is a base class used to represent
2682/// VP_GATHER and VP_SCATTER nodes
2683///
2684class VPGatherScatterSDNode : public MemSDNode {
2685public:
friend class SelectionDAG;

VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                      MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2693, __extension__
 __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// VPGatherSDNode  (Chain, base, index, scale, mask, vlen)
// VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
}
const SDValue &getIndex() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
}
const SDValue &getScale() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
}
const SDValue &getMask() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
}
const SDValue &getVectorLength() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER ||
         N->getOpcode() == ISD::VP_SCATTER;
}
2733};

2735/// This class is used to represent an VP_GATHER node
2736///
2737class VPGatherSDNode : public VPGatherScatterSDNode {
2738public:
friend class SelectionDAG;

VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
               MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER;
}
2749};

2751/// This class is used to represent an VP_SCATTER node
2752///
2753class VPScatterSDNode : public VPGatherScatterSDNode {
2754public:
friend class SelectionDAG;

VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_SCATTER;
}
2767};

2769/// This is a base class used to represent
2770/// MGATHER and MSCATTER nodes
2771///
2772class MaskedGatherScatterSDNode : public MemSDNode {
2773public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2781, __extension__
 __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
void setIndexType(ISD::MemIndexType IndexType) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2813};

2815/// This class is used to represent an MGATHER node
2816///
2817class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2818public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2838};

2840/// This class is used to represent an MSCATTER node
2841///
2842class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2843public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2864};

2866/// An SDNode that represents everything that will be needed
2867/// to construct a MachineInstr. These nodes are created during the
2868/// instruction selection proper phase.
2869///
2870/// Note that the only supported way to set the `memoperands` is by calling the
2871/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2872/// inside the DAG rather than in the node.
2873class MachineSDNode : public SDNode {
2874private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2902public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2928};

2930/// An SDNode that records if a register contains a value that is guaranteed to
2931/// be aligned accordingly.
2932class AssertAlignSDNode : public SDNode {
Align Alignment;

2935public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2944};

2946class SDNodeIterator {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2952public:
using iterator_category = std::forward_iterator_tag;
using value_type = SDNode;
using difference_type = std::ptrdiff_t;
using pointer = value_type *;
using reference = value_type &;

bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2978, __extension__
 __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2978, __extension__
 __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
2989};

2991template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
3004};

3006/// A representation of the largest SDNode, for use in sizeof().
3007///
3008/// This needs to be a union because the largest node differs on 32 bit systems
3009/// with 4 and 8 byte pointer alignment, respectively.
3010using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

3015/// The SDNode class with the greatest alignment requirement.
3016using MostAlignedSDNode = GlobalAddressSDNode;

3018namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

3096} // end namespace ISD

3098} // end namespace llvm

3100#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H