/build/source/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Bug Summary

File:	build/source/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:	line 1142, column 10 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAGBuilder.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/source/llvm/lib/CodeGen/SelectionDAG -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

/build/source/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

→

1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//

13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/Analysis/AliasAnalysis.h"
24#include "llvm/Analysis/BranchProbabilityInfo.h"
25#include "llvm/Analysis/ConstantFolding.h"
26#include "llvm/Analysis/Loads.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/CodeGen/Analysis.h"
32#include "llvm/CodeGen/AssignmentTrackingAnalysis.h"
33#include "llvm/CodeGen/CodeGenCommonISel.h"
34#include "llvm/CodeGen/FunctionLoweringInfo.h"
35#include "llvm/CodeGen/GCMetadata.h"
36#include "llvm/CodeGen/MachineBasicBlock.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineInstrBuilder.h"
40#include "llvm/CodeGen/MachineInstrBundleIterator.h"
41#include "llvm/CodeGen/MachineMemOperand.h"
42#include "llvm/CodeGen/MachineModuleInfo.h"
43#include "llvm/CodeGen/MachineOperand.h"
44#include "llvm/CodeGen/MachineRegisterInfo.h"
45#include "llvm/CodeGen/RuntimeLibcalls.h"
46#include "llvm/CodeGen/SelectionDAG.h"
47#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
48#include "llvm/CodeGen/StackMaps.h"
49#include "llvm/CodeGen/SwiftErrorValueTracking.h"
50#include "llvm/CodeGen/TargetFrameLowering.h"
51#include "llvm/CodeGen/TargetInstrInfo.h"
52#include "llvm/CodeGen/TargetOpcodes.h"
53#include "llvm/CodeGen/TargetRegisterInfo.h"
54#include "llvm/CodeGen/TargetSubtargetInfo.h"
55#include "llvm/CodeGen/WinEHFuncInfo.h"
56#include "llvm/IR/Argument.h"
57#include "llvm/IR/Attributes.h"
58#include "llvm/IR/BasicBlock.h"
59#include "llvm/IR/CFG.h"
60#include "llvm/IR/CallingConv.h"
61#include "llvm/IR/Constant.h"
62#include "llvm/IR/ConstantRange.h"
63#include "llvm/IR/Constants.h"
64#include "llvm/IR/DataLayout.h"
65#include "llvm/IR/DebugInfo.h"
66#include "llvm/IR/DebugInfoMetadata.h"
67#include "llvm/IR/DerivedTypes.h"
68#include "llvm/IR/DiagnosticInfo.h"
69#include "llvm/IR/EHPersonalities.h"
70#include "llvm/IR/Function.h"
71#include "llvm/IR/GetElementPtrTypeIterator.h"
72#include "llvm/IR/InlineAsm.h"
73#include "llvm/IR/InstrTypes.h"
74#include "llvm/IR/Instructions.h"
75#include "llvm/IR/IntrinsicInst.h"
76#include "llvm/IR/Intrinsics.h"
77#include "llvm/IR/IntrinsicsAArch64.h"
78#include "llvm/IR/IntrinsicsWebAssembly.h"
79#include "llvm/IR/LLVMContext.h"
80#include "llvm/IR/Metadata.h"
81#include "llvm/IR/Module.h"
82#include "llvm/IR/Operator.h"
83#include "llvm/IR/PatternMatch.h"
84#include "llvm/IR/Statepoint.h"
85#include "llvm/IR/Type.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/MC/MCContext.h"
89#include "llvm/Support/AtomicOrdering.h"
90#include "llvm/Support/Casting.h"
91#include "llvm/Support/CommandLine.h"
92#include "llvm/Support/Compiler.h"
93#include "llvm/Support/Debug.h"
94#include "llvm/Support/MathExtras.h"
95#include "llvm/Support/raw_ostream.h"
96#include "llvm/Target/TargetIntrinsicInfo.h"
97#include "llvm/Target/TargetMachine.h"
98#include "llvm/Target/TargetOptions.h"
99#include "llvm/TargetParser/Triple.h"
100#include "llvm/Transforms/Utils/Local.h"
101#include <cstddef>
102#include <iterator>
103#include <limits>
104#include <optional>
105#include <tuple>

107using namespace llvm;
108using namespace PatternMatch;
109using namespace SwitchCG;

111#define DEBUG_TYPE"isel" "isel"

113/// LimitFloatPrecision - Generate low-precision inline sequences for
114/// some float libcalls (6, 8 or 12 bits).
115static unsigned LimitFloatPrecision;

117static cl::opt<bool>
  InsertAssertAlign("insert-assert-align", cl::init(true),
                    cl::desc("Insert the experimental `assertalign` node."),
                    cl::ReallyHidden);

122static cl::opt<unsigned, true>
  LimitFPPrecision("limit-float-precision",
                   cl::desc("Generate low-precision inline sequences "
                            "for some float libcalls"),
                   cl::location(LimitFloatPrecision), cl::Hidden,
                   cl::init(0));

129static cl::opt<unsigned> SwitchPeelThreshold(
  "switch-peel-threshold", cl::Hidden, cl::init(66),
  cl::desc("Set the case probability threshold for peeling the case from a "
           "switch statement. A value greater than 100 will void this "
           "optimization"));

135// Limit the width of DAG chains. This is important in general to prevent
136// DAG-based analysis from blowing up. For example, alias analysis and
137// load clustering may not complete in reasonable time. It is difficult to
138// recognize and avoid this situation within each individual analysis, and
139// future analyses are likely to have the same behavior. Limiting DAG width is
140// the safe approach and will be especially important with global DAGs.
141//
142// MaxParallelChains default is arbitrarily high to avoid affecting
143// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
144// sequence over this should have been converted to llvm.memcpy by the
145// frontend. It is easy to induce this behavior with .ll code such as:
146// %buffer = alloca [4096 x i8]
147// %data = load [4096 x i8]* %argPtr
148// store [4096 x i8] %data, [4096 x i8]* %buffer
149static const unsigned MaxParallelChains = 64;

151static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
                                    const SDValue *Parts, unsigned NumParts,
                                    MVT PartVT, EVT ValueVT, const Value *V,
                                    std::optional<CallingConv::ID> CC);

156/// getCopyFromParts - Create a value that contains the specified legal parts
157/// combined into the value they represent.  If the parts combine to a type
158/// larger than ValueVT then AssertOp can be used to specify whether the extra
159/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
160/// (ISD::AssertSext).
161static SDValue
162getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
               unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
               std::optional<CallingConv::ID> CC = std::nullopt,
               std::optional<ISD::NodeType> AssertOp = std::nullopt) {
// Let the target assemble the parts if it wants to
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
                                                 PartVT, ValueVT, CC))
  return Val;

if (ValueVT.isVector())
  return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
                                CC);

assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 176
, __extension__ __PRETTY_FUNCTION__));
SDValue Val = Parts[0];

if (NumParts > 1) {
  // Assemble the value from multiple parts.
  if (ValueVT.isInteger()) {
    unsigned PartBits = PartVT.getSizeInBits();
    unsigned ValueBits = ValueVT.getSizeInBits();

    // Assemble the power of 2 part.
    unsigned RoundParts = llvm::bit_floor(NumParts);
    unsigned RoundBits = PartBits * RoundParts;
    EVT RoundVT = RoundBits == ValueBits ?
      ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
    SDValue Lo, Hi;

    EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);

    if (RoundParts > 2) {
      Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
                            PartVT, HalfVT, V);
      Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
                            RoundParts / 2, PartVT, HalfVT, V);
    } else {
      Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
      Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
    }

    if (DAG.getDataLayout().isBigEndian())
      std::swap(Lo, Hi);

    Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);

    if (RoundParts < NumParts) {
      // Assemble the trailing non-power-of-2 part.
      unsigned OddParts = NumParts - RoundParts;
      EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
      Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
                            OddVT, V, CC);

      // Combine the round and odd parts.
      Lo = Val;
      if (DAG.getDataLayout().isBigEndian())
        std::swap(Lo, Hi);
      EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
      Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
      Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
                       DAG.getConstant(Lo.getValueSizeInBits(), DL,
                                       TLI.getShiftAmountTy(
                                           TotalVT, DAG.getDataLayout())));
      Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
      Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
    }
  } else if (PartVT.isFloatingPoint()) {
    // FP split into multiple FP parts (for ppcf128)
    assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
 PartVT == MVT::f64 && "Unexpected split") ? void (0)
 : __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 232
, __extension__ __PRETTY_FUNCTION__))
           "Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
 PartVT == MVT::f64 && "Unexpected split") ? void (0)
 : __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 232
, __extension__ __PRETTY_FUNCTION__));
    SDValue Lo, Hi;
    Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
    Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
    if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
      std::swap(Lo, Hi);
    Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
  } else {
    // FP split into integer parts (soft fp)
    assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() &&
 PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 242
, __extension__ __PRETTY_FUNCTION__))
           !PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() &&
 PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 242
, __extension__ __PRETTY_FUNCTION__));
    EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
    Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
  }
}

// There is now one part, held in Val.  Correct it to match ValueVT.
// PartEVT is the type of the register class that holds the value.
// ValueVT is the type of the inline asm operation.
EVT PartEVT = Val.getValueType();

if (PartEVT == ValueVT)
  return Val;

if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
    ValueVT.bitsLT(PartEVT)) {
  // For an FP value in an integer part, we need to truncate to the right
  // width first.
  PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
  Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
}

// Handle types that have the same size.
if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

// Handle types with different sizes.
if (PartEVT.isInteger() && ValueVT.isInteger()) {
  if (ValueVT.bitsLT(PartEVT)) {
    // For a truncate, see if we have any information to
    // indicate whether the truncated bits will always be
    // zero or sign-extension.
    if (AssertOp)
      Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
                        DAG.getValueType(ValueVT));
    return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
  }
  return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
}

if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
  // FP_ROUND's are always exact here.
  if (ValueVT.bitsLT(Val.getValueType()))
    return DAG.getNode(
        ISD::FP_ROUND, DL, ValueVT, Val,
        DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));

  return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
}

// Handle MMX to a narrower integer type by bitcasting MMX to integer and
// then truncating.
if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
    ValueVT.bitsLT(PartEVT)) {
  Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
  return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
}

report_fatal_error("Unknown mismatch in getCopyFromParts!");
301}

303static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
                                            const Twine &ErrMsg) {
const Instruction *I = dyn_cast_or_null<Instruction>(V);
if (!V)
  return Ctx.emitError(ErrMsg);

const char *AsmError = ", possible invalid constraint for vector type";
if (const CallInst *CI = dyn_cast<CallInst>(I))
  if (CI->isInlineAsm())
    return Ctx.emitError(I, ErrMsg + AsmError);

return Ctx.emitError(I, ErrMsg);
315}

317/// getCopyFromPartsVector - Create a value that contains the specified legal
318/// parts combined into the value they represent.  If the parts combine to a
319/// type larger than ValueVT then AssertOp can be used to specify whether the
320/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
321/// ValueVT (ISD::AssertSext).
322static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
                                    const SDValue *Parts, unsigned NumParts,
                                    MVT PartVT, EVT ValueVT, const Value *V,
                                    std::optional<CallingConv::ID> CallConv) {
assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 326
, __extension__ __PRETTY_FUNCTION__));
assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 327
, __extension__ __PRETTY_FUNCTION__));
const bool IsABIRegCopy = CallConv.has_value();

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue Val = Parts[0];

// Handle a multi-element vector.
if (NumParts > 1) {
  EVT IntermediateVT;
  MVT RegisterVT;
  unsigned NumIntermediates;
  unsigned NumRegs;

  if (IsABIRegCopy) {
    NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
        *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
        NumIntermediates, RegisterVT);
  } else {
    NumRegs =
        TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
                                   NumIntermediates, RegisterVT);
  }

  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 350
, __extension__ __PRETTY_FUNCTION__));
  NumParts = NumRegs; // Silence a compiler warning.
  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 352
, __extension__ __PRETTY_FUNCTION__));
  assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355
, __extension__ __PRETTY_FUNCTION__))
         Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355
, __extension__ __PRETTY_FUNCTION__))
         "Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 355
, __extension__ __PRETTY_FUNCTION__));

  // Assemble the parts into intermediate operands.
  SmallVector<SDValue, 8> Ops(NumIntermediates);
  if (NumIntermediates == NumParts) {
    // If the register was not expanded, truncate or copy the value,
    // as appropriate.
    for (unsigned i = 0; i != NumParts; ++i)
      Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
                                PartVT, IntermediateVT, V, CallConv);
  } else if (NumParts > 0) {
    // If the intermediate type was expanded, build the intermediate
    // operands from the parts.
    assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
 "Must expand into a divisible number of parts!") ? void (0) :
 __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 369
, __extension__ __PRETTY_FUNCTION__))
           "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
 "Must expand into a divisible number of parts!") ? void (0) :
 __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 369
, __extension__ __PRETTY_FUNCTION__));
    unsigned Factor = NumParts / NumIntermediates;
    for (unsigned i = 0; i != NumIntermediates; ++i)
      Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
                                PartVT, IntermediateVT, V, CallConv);
  }

  // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
  // intermediate operands.
  EVT BuiltVectorTy =
      IntermediateVT.isVector()
          ? EVT::getVectorVT(
                *DAG.getContext(), IntermediateVT.getScalarType(),
                IntermediateVT.getVectorElementCount() * NumParts)
          : EVT::getVectorVT(*DAG.getContext(),
                             IntermediateVT.getScalarType(),
                             NumIntermediates);
  Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
                                              : ISD::BUILD_VECTOR,
                    DL, BuiltVectorTy, Ops);
}

// There is now one part, held in Val.  Correct it to match ValueVT.
EVT PartEVT = Val.getValueType();

if (PartEVT == ValueVT)
  return Val;

if (PartEVT.isVector()) {
  // Vector/Vector bitcast.
  if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
    return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

  // If the parts vector has more elements than the value vector, then we
  // have a vector widening case (e.g. <2 x float> -> <4 x float>).
  // Extract the elements we want.
  if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
    assert((PartEVT.getVectorElementCount().getKnownMinValue() >(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
 (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))
            ValueVT.getVectorElementCount().getKnownMinValue()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
 (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))
           (PartEVT.getVectorElementCount().isScalable() ==(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
 (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))
            ValueVT.getVectorElementCount().isScalable()) &&(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
 (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__))
           "Cannot narrow, it would be a lossy transformation")(static_cast <bool> ((PartEVT.getVectorElementCount().getKnownMinValue
() > ValueVT.getVectorElementCount().getKnownMinValue()) &&
 (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount
().isScalable()) && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("(PartEVT.getVectorElementCount().getKnownMinValue() > ValueVT.getVectorElementCount().getKnownMinValue()) && (PartEVT.getVectorElementCount().isScalable() == ValueVT.getVectorElementCount().isScalable()) && \"Cannot narrow, it would be a lossy transformation\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 410
, __extension__ __PRETTY_FUNCTION__));
    PartEVT =
        EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(),
                         ValueVT.getVectorElementCount());
    Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
                      DAG.getVectorIdxConstant(0, DL));
    if (PartEVT == ValueVT)
      return Val;
    if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
      return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
  }

  // Promoted vector extract
  return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
}

// Trivial bitcast if the types are the same size and the destination
// vector type is legal.
if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
    TLI.isTypeLegal(ValueVT))
  return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);

if (ValueVT.getVectorNumElements() != 1) {
   // Certain ABIs require that vectors are passed as integers. For vectors
   // are the same size, this is an obvious bitcast.
   if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
   } else if (ValueVT.bitsLT(PartEVT)) {
     const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
     EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
     // Drop the extra bits.
     Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
     return DAG.getBitcast(ValueVT, Val);
   }

   diagnosePossiblyInvalidConstraint(
       *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
   return DAG.getUNDEF(ValueVT);
}

// Handle cases such as i8 -> <1 x i1>
EVT ValueSVT = ValueVT.getVectorElementType();
if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
  unsigned ValueSize = ValueSVT.getSizeInBits();
  if (ValueSize == PartEVT.getSizeInBits()) {
    Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
  } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
    // It's possible a scalar floating point type gets softened to integer and
    // then promoted to a larger integer. If PartEVT is the larger integer
    // we need to truncate it and then bitcast to the FP type.
    assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types")(static_cast <bool> (ValueSVT.bitsLT(PartEVT) &&
 "Unexpected types") ? void (0) : __assert_fail ("ValueSVT.bitsLT(PartEVT) && \"Unexpected types\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 460
, __extension__ __PRETTY_FUNCTION__));
    EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
    Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
    Val = DAG.getBitcast(ValueSVT, Val);
  } else {
    Val = ValueVT.isFloatingPoint()
              ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
              : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
  }
}

return DAG.getBuildVector(ValueVT, DL, Val);
472}

474static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
                               SDValue Val, SDValue *Parts, unsigned NumParts,
                               MVT PartVT, const Value *V,
                               std::optional<CallingConv::ID> CallConv);

479/// getCopyToParts - Create a series of nodes that contain the specified value
480/// split into legal parts.  If the parts contain more bits than Val, then, for
481/// integers, ExtendKind can be used to specify how to generate the extra bits.
482static void
483getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
             unsigned NumParts, MVT PartVT, const Value *V,
             std::optional<CallingConv::ID> CallConv = std::nullopt,
             ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
// Let the target split the parts if it wants to
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
                                    CallConv))
  return;
EVT ValueVT = Val.getValueType();

// Handle the vector case separately.
if (ValueVT.isVector())
  return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
                              CallConv);

unsigned OrigNumParts = NumParts;
assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
 : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 501
, __extension__ __PRETTY_FUNCTION__))
       "Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
 : __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 501
, __extension__ __PRETTY_FUNCTION__));

if (NumParts == 0)
  return;

assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere"
) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 506
, __extension__ __PRETTY_FUNCTION__));
EVT PartEVT = PartVT;
if (PartEVT == ValueVT) {
  assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 509
, __extension__ __PRETTY_FUNCTION__));
  Parts[0] = Val;
  return;
}

unsigned PartBits = PartVT.getSizeInBits();
if (NumParts * PartBits > ValueVT.getSizeInBits()) {
  // If the parts cover more bits than the value has, promote the value.
  if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
    assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 518
, __extension__ __PRETTY_FUNCTION__));
    Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
  } else {
    if (ValueVT.isFloatingPoint()) {
      // FP values need to be bitcast, then extended if they are being put
      // into a larger container.
      ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
      Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
    }
    assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529
, __extension__ __PRETTY_FUNCTION__))
           ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529
, __extension__ __PRETTY_FUNCTION__))
           "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 529
, __extension__ __PRETTY_FUNCTION__));
    ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
    Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
    if (PartVT == MVT::x86mmx)
      Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
  }
} else if (PartBits == ValueVT.getSizeInBits()) {
  // Different types of the same size.
  assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT !=
 ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 537
, __extension__ __PRETTY_FUNCTION__));
  Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
} else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
  // If the parts cover less bits than value has, truncate the value.
  assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543
, __extension__ __PRETTY_FUNCTION__))
         ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543
, __extension__ __PRETTY_FUNCTION__))
         "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 543
, __extension__ __PRETTY_FUNCTION__));
  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
  if (PartVT == MVT::x86mmx)
    Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
}

// The value may have changed - recompute ValueVT.
ValueVT = Val.getValueType();
assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
 (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 553
, __extension__ __PRETTY_FUNCTION__))
       "Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
 (0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 553
, __extension__ __PRETTY_FUNCTION__));

if (NumParts == 1) {
  if (PartEVT != ValueVT) {
    diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
                                      "scalar-to-vector conversion failed");
    Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
  }

  Parts[0] = Val;
  return;
}

// Expand the value into multiple parts.
if (NumParts & (NumParts - 1)) {
  // The number of parts is not a power of 2.  Split off and copy the tail.
  assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
 (0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 570
, __extension__ __PRETTY_FUNCTION__))
         "Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
 (0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 570
, __extension__ __PRETTY_FUNCTION__));
  unsigned RoundParts = llvm::bit_floor(NumParts);
  unsigned RoundBits = RoundParts * PartBits;
  unsigned OddParts = NumParts - RoundParts;
  SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
    DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));

  getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
                 CallConv);

  if (DAG.getDataLayout().isBigEndian())
    // The odd parts were reversed by getCopyToParts - unreverse them.
    std::reverse(Parts + RoundParts, Parts + NumParts);

  NumParts = RoundParts;
  ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
  Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
}

// The number of parts is a power of 2.  Repeatedly bisect the value using
// EXTRACT_ELEMENT.
Parts[0] = DAG.getNode(ISD::BITCAST, DL,
                       EVT::getIntegerVT(*DAG.getContext(),
                                         ValueVT.getSizeInBits()),
                       Val);

for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
  for (unsigned i = 0; i < NumParts; i += StepSize) {
    unsigned ThisBits = StepSize * PartBits / 2;
    EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
    SDValue &Part0 = Parts[i];
    SDValue &Part1 = Parts[i+StepSize/2];

    Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
                        ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
    Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
                        ThisVT, Part0, DAG.getIntPtrConstant(0, DL));

    if (ThisBits == PartBits && ThisVT != PartVT) {
      Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
      Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
    }
  }
}

if (DAG.getDataLayout().isBigEndian())
  std::reverse(Parts, Parts + OrigNumParts);
617}

619static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val,
                                   const SDLoc &DL, EVT PartVT) {
if (!PartVT.isVector())
  return SDValue();

EVT ValueVT = Val.getValueType();
ElementCount PartNumElts = PartVT.getVectorElementCount();
ElementCount ValueNumElts = ValueVT.getVectorElementCount();

// We only support widening vectors with equivalent element types and
// fixed/scalable properties. If a target needs to widen a fixed-length type
// to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
    PartNumElts.isScalable() != ValueNumElts.isScalable() ||
    PartVT.getVectorElementType() != ValueVT.getVectorElementType())
  return SDValue();

// Widening a scalable vector to another scalable vector is done by inserting
// the vector into a larger undef one.
if (PartNumElts.isScalable())
  return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
                     Val, DAG.getVectorIdxConstant(0, DL));

EVT ElementVT = PartVT.getVectorElementType();
// Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
// undef elements.
SmallVector<SDValue, 16> Ops;
DAG.ExtractVectorElements(Val, Ops);
SDValue EltUndef = DAG.getUNDEF(ElementVT);
Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);

// FIXME: Use CONCAT for 2x -> 4x.
return DAG.getBuildVector(PartVT, DL, Ops);
652}

654/// getCopyToPartsVector - Create a series of nodes that contain the specified
655/// value split into legal parts.
656static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
                               SDValue Val, SDValue *Parts, unsigned NumParts,
                               MVT PartVT, const Value *V,
                               std::optional<CallingConv::ID> CallConv) {
EVT ValueVT = Val.getValueType();
assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 661
, __extension__ __PRETTY_FUNCTION__));
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const bool IsABIRegCopy = CallConv.has_value();

if (NumParts == 1) {
  EVT PartEVT = PartVT;
  if (PartEVT == ValueVT) {
    // Nothing to do.
  } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
    // Bitconvert vector->vector case.
    Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
  } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
    Val = Widened;
  } else if (PartVT.isVector() &&
             PartEVT.getVectorElementType().bitsGE(
                 ValueVT.getVectorElementType()) &&
             PartEVT.getVectorElementCount() ==
                 ValueVT.getVectorElementCount()) {

    // Promoted vector extract
    Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
  } else if (PartEVT.isVector() &&
             PartEVT.getVectorElementType() !=
                 ValueVT.getVectorElementType() &&
             TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
                 TargetLowering::TypeWidenVector) {
    // Combination of widening and promotion.
    EVT WidenVT =
        EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(),
                         PartVT.getVectorElementCount());
    SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
    Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
  } else {
    // Don't extract an integer from a float vector. This can happen if the
    // FP type gets softened to integer and then promoted. The promotion
    // prevents it from being picked up by the earlier bitcast case.
    if (ValueVT.getVectorElementCount().isScalar() &&
        (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
      Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
                        DAG.getVectorIdxConstant(0, DL));
    } else {
      uint64_t ValueSize = ValueVT.getFixedSizeInBits();
      assert(PartVT.getFixedSizeInBits() > ValueSize &&(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
 && "lossy conversion of vector to scalar type") ? void
 (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 704
, __extension__ __PRETTY_FUNCTION__))
             "lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getFixedSizeInBits() > ValueSize
 && "lossy conversion of vector to scalar type") ? void
 (0) : __assert_fail ("PartVT.getFixedSizeInBits() > ValueSize && \"lossy conversion of vector to scalar type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 704
, __extension__ __PRETTY_FUNCTION__));
      EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
      Val = DAG.getBitcast(IntermediateType, Val);
      Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
    }
  }

  assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT &&
 "Unexpected vector part value type") ? void (0) : __assert_fail
 ("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 711
, __extension__ __PRETTY_FUNCTION__));
  Parts[0] = Val;
  return;
}

// Handle a multi-element vector.
EVT IntermediateVT;
MVT RegisterVT;
unsigned NumIntermediates;
unsigned NumRegs;
if (IsABIRegCopy) {
  NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
      *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
      RegisterVT);
} else {
  NumRegs =
      TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
                                 NumIntermediates, RegisterVT);
}

assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 731
, __extension__ __PRETTY_FUNCTION__));
NumParts = NumRegs; // Silence a compiler warning.
assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 733
, __extension__ __PRETTY_FUNCTION__));

assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&(static_cast <bool> (IntermediateVT.isScalableVector() ==
 ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 736
, __extension__ __PRETTY_FUNCTION__))
       "Mixing scalable and fixed vectors when copying in parts")(static_cast <bool> (IntermediateVT.isScalableVector() ==
 ValueVT.isScalableVector() && "Mixing scalable and fixed vectors when copying in parts"
) ? void (0) : __assert_fail ("IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && \"Mixing scalable and fixed vectors when copying in parts\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 736
, __extension__ __PRETTY_FUNCTION__));

std::optional<ElementCount> DestEltCnt;

if (IntermediateVT.isVector())
  DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
else
  DestEltCnt = ElementCount::getFixed(NumIntermediates);

EVT BuiltVectorTy = EVT::getVectorVT(
    *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);

if (ValueVT == BuiltVectorTy) {
  // Nothing to do.
} else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
  // Bitconvert vector->vector case.
  Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
} else {
  if (BuiltVectorTy.getVectorElementType().bitsGT(
          ValueVT.getVectorElementType())) {
    // Integer promotion.
    ValueVT = EVT::getVectorVT(*DAG.getContext(),
                               BuiltVectorTy.getVectorElementType(),
                               ValueVT.getVectorElementCount());
    Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
  }

  if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
    Val = Widened;
  }
}

assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type")(static_cast <bool> (Val.getValueType() == BuiltVectorTy
 && "Unexpected vector value type") ? void (0) : __assert_fail
 ("Val.getValueType() == BuiltVectorTy && \"Unexpected vector value type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 768
, __extension__ __PRETTY_FUNCTION__));

// Split the vector into intermediate operands.
SmallVector<SDValue, 8> Ops(NumIntermediates);
for (unsigned i = 0; i != NumIntermediates; ++i) {
  if (IntermediateVT.isVector()) {
    // This does something sensible for scalable vectors - see the
    // definition of EXTRACT_SUBVECTOR for further details.
    unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
    Ops[i] =
        DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
                    DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
  } else {
    Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
                         DAG.getVectorIdxConstant(i, DL));
  }
}

// Split the intermediate operands into legal parts.
if (NumParts == NumIntermediates) {
  // If the register was not expanded, promote or copy the value,
  // as appropriate.
  for (unsigned i = 0; i != NumParts; ++i)
    getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
} else if (NumParts > 0) {
  // If the intermediate type was expanded, split each the value into
  // legal parts.
  assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero"
) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 795
, __extension__ __PRETTY_FUNCTION__));
  assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
 "Must expand into a divisible number of parts!") ? void (0) :
 __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 797
, __extension__ __PRETTY_FUNCTION__))
         "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
 "Must expand into a divisible number of parts!") ? void (0) :
 __assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 797
, __extension__ __PRETTY_FUNCTION__));
  unsigned Factor = NumParts / NumIntermediates;
  for (unsigned i = 0; i != NumIntermediates; ++i)
    getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
                   CallConv);
}
803}

805RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
                         EVT valuevt, std::optional<CallingConv::ID> CC)
  : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
    RegCount(1, regs.size()), CallConv(CC) {}

810RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
                         const DataLayout &DL, unsigned Reg, Type *Ty,
                         std::optional<CallingConv::ID> CC) {
ComputeValueVTs(TLI, DL, Ty, ValueVTs);

CallConv = CC;

for (EVT ValueVT : ValueVTs) {
  unsigned NumRegs =
      isABIMangled()
          ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
          : TLI.getNumRegisters(Context, ValueVT);
  MVT RegisterVT =
      isABIMangled()
          ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
          : TLI.getRegisterType(Context, ValueVT);
  for (unsigned i = 0; i != NumRegs; ++i)
    Regs.push_back(Reg + i);
  RegVTs.push_back(RegisterVT);
  RegCount.push_back(NumRegs);
  Reg += NumRegs;
}
832}

834SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
                                    FunctionLoweringInfo &FuncInfo,
                                    const SDLoc &dl, SDValue &Chain,
                                    SDValue *Glue, const Value *V) const {
// A Value with type {} or [0 x %t] needs no registers.
if (ValueVTs.empty())
  return SDValue();

const TargetLowering &TLI = DAG.getTargetLoweringInfo();

// Assemble the legal parts into the final values.
SmallVector<SDValue, 4> Values(ValueVTs.size());
SmallVector<SDValue, 8> Parts;
for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
  // Copy the legal parts from the registers.
  EVT ValueVT = ValueVTs[Value];
  unsigned NumRegs = RegCount[Value];
  MVT RegisterVT = isABIMangled()
                       ? TLI.getRegisterTypeForCallingConv(
                             *DAG.getContext(), *CallConv, RegVTs[Value])
                       : RegVTs[Value];

  Parts.resize(NumRegs);
  for (unsigned i = 0; i != NumRegs; ++i) {
    SDValue P;
    if (!Glue) {
      P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
    } else {
      P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
      *Glue = P.getValue(2);
    }

    Chain = P.getValue(1);
    Parts[i] = P;

    // If the source register was virtual and if we know something about it,
    // add an assert node.
    if (!Register::isVirtualRegister(Regs[Part + i]) ||
        !RegisterVT.isInteger())
      continue;

    const FunctionLoweringInfo::LiveOutInfo *LOI =
      FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
    if (!LOI)
      continue;

    unsigned RegSize = RegisterVT.getScalarSizeInBits();
    unsigned NumSignBits = LOI->NumSignBits;
    unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();

    if (NumZeroBits == RegSize) {
      // The current value is a zero.
      // Explicitly express that as it would be easier for
      // optimizations to kick in.
      Parts[i] = DAG.getConstant(0, dl, RegisterVT);
      continue;
    }

    // FIXME: We capture more information than the dag can represent.  For
    // now, just use the tightest assertzext/assertsext possible.
    bool isSExt;
    EVT FromVT(MVT::Other);
    if (NumZeroBits) {
      FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
      isSExt = false;
    } else if (NumSignBits > 1) {
      FromVT =
          EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
      isSExt = true;
    } else {
      continue;
    }
    // Add an assertion node.
    assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) :
 __assert_fail ("FromVT != MVT::Other", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 907, __extension__ __PRETTY_FUNCTION__));
    Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
                           RegisterVT, P, DAG.getValueType(FromVT));
  }

  Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
                                   RegisterVT, ValueVT, V, CallConv);
  Part += NumRegs;
  Parts.clear();
}

return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
919}

921void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
                               const SDLoc &dl, SDValue &Chain, SDValue *Glue,
                               const Value *V,
                               ISD::NodeType PreferredExtendType) const {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
ISD::NodeType ExtendKind = PreferredExtendType;

// Get the list of the values's legal parts.
unsigned NumRegs = Regs.size();
SmallVector<SDValue, 8> Parts(NumRegs);
for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
  unsigned NumParts = RegCount[Value];

  MVT RegisterVT = isABIMangled()
                       ? TLI.getRegisterTypeForCallingConv(
                             *DAG.getContext(), *CallConv, RegVTs[Value])
                       : RegVTs[Value];

  if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
    ExtendKind = ISD::ZERO_EXTEND;

  getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
                 NumParts, RegisterVT, V, CallConv, ExtendKind);
  Part += NumParts;
}

// Copy the parts into the registers.
SmallVector<SDValue, 8> Chains(NumRegs);
for (unsigned i = 0; i != NumRegs; ++i) {
  SDValue Part;
  if (!Glue) {
    Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
  } else {
    Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
    *Glue = Part.getValue(1);
  }

  Chains[i] = Part.getValue(0);
}

if (NumRegs == 1 || Glue)
  // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
  // flagged to it. That is the CopyToReg nodes and the user are considered
  // a single scheduling unit. If we create a TokenFactor and return it as
  // chain, then the TokenFactor is both a predecessor (operand) of the
  // user as well as a successor (the TF operands are flagged to the user).
  // c1, f1 = CopyToReg
  // c2, f2 = CopyToReg
  // c3     = TokenFactor c1, c2
  // ...
  //        = op c3, ..., f2
  Chain = Chains[NumRegs-1];
else
  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
975}

977void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
                                      unsigned MatchingIdx, const SDLoc &dl,
                                      SelectionDAG &DAG,
                                      std::vector<SDValue> &Ops) const {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
if (HasMatching)
  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
  // Put the register class of the virtual registers in the flag word.  That
  // way, later passes can recompute register class constraints for inline
  // assembly as well as normal instructions.
  // Don't do this for tied operands that can use the regclass information
  // from the def.
  const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
  const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
  Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
}

SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
Ops.push_back(Res);

if (Code == InlineAsm::Kind_Clobber) {
  // Clobbers should always have a 1:1 mapping with registers, and may
  // reference registers that have illegal (e.g. vector) types. Hence, we
  // shouldn't try to apply any sort of splitting logic to them.
  assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() &&
 Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1005
, __extension__ __PRETTY_FUNCTION__))
         "No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() &&
 Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1005
, __extension__ __PRETTY_FUNCTION__));
  Register SP = TLI.getStackPointerRegisterToSaveRestore();
  (void)SP;
  for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
    Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
    assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__))
        (Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__))
         DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__))
        "If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1013
, __extension__ __PRETTY_FUNCTION__));
  }
  return;
}

for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
  MVT RegisterVT = RegVTs[Value];
  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
                                         RegisterVT);
  for (unsigned i = 0; i != NumRegs; ++i) {
    assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected"
) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1023
, __extension__ __PRETTY_FUNCTION__));
    unsigned TheReg = Regs[Reg++];
    Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
  }
}
1028}

1030SmallVector<std::pair<unsigned, TypeSize>, 4>
1031RegsForValue::getRegsAndSizes() const {
SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
unsigned I = 0;
for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
  unsigned RegCount = std::get<0>(CountAndVT);
  MVT RegisterVT = std::get<1>(CountAndVT);
  TypeSize RegisterSize = RegisterVT.getSizeInBits();
  for (unsigned E = I + RegCount; I != E; ++I)
    OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
}
return OutVec;
1042}

1044void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
                             AssumptionCache *ac,
                             const TargetLibraryInfo *li) {
AA = aa;
AC = ac;
GFI = gfi;
LibInfo = li;
Context = DAG.getContext();
LPadToCallSiteMap.clear();
SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
    *DAG.getMachineFunction().getFunction().getParent());
1056}

1058void SelectionDAGBuilder::clear() {
NodeMap.clear();
UnusedArgNodeMap.clear();
PendingLoads.clear();
PendingExports.clear();
PendingConstrainedFP.clear();
PendingConstrainedFPStrict.clear();
CurInst = nullptr;
HasTailCall = false;
SDNodeOrder = LowestSDNodeOrder;
StatepointLowering.clear();
1069}

1071void SelectionDAGBuilder::clearDanglingDebugInfo() {
DanglingDebugInfoMap.clear();
1073}

1075// Update DAG root to include dependencies on Pending chains.
1076SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
SDValue Root = DAG.getRoot();

if (Pending.empty())
  return Root;

// Add current root to PendingChains, unless we already indirectly
// depend on it.
if (Root.getOpcode() != ISD::EntryToken) {
  unsigned i = 0, e = Pending.size();
  for (; i != e; ++i) {
    assert(Pending[i].getNode()->getNumOperands() > 1)(static_cast <bool> (Pending[i].getNode()->getNumOperands
() > 1) ? void (0) : __assert_fail ("Pending[i].getNode()->getNumOperands() > 1"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1087
, __extension__ __PRETTY_FUNCTION__));
    if (Pending[i].getNode()->getOperand(0) == Root)
      break;  // Don't add the root if we already indirectly depend on it.
  }

  if (i == e)
    Pending.push_back(Root);
}

if (Pending.size() == 1)
  Root = Pending[0];
else
  Root = DAG.getTokenFactor(getCurSDLoc(), Pending);

DAG.setRoot(Root);
Pending.clear();
return Root;
1104}

1106SDValue SelectionDAGBuilder::getMemoryRoot() {
return updateRoot(PendingLoads);
1108}

1110SDValue SelectionDAGBuilder::getRoot() {
// Chain up all pending constrained intrinsics together with all
// pending loads, by simply appending them to PendingLoads and
// then calling getMemoryRoot().
PendingLoads.reserve(PendingLoads.size() +
                     PendingConstrainedFP.size() +
                     PendingConstrainedFPStrict.size());
PendingLoads.append(PendingConstrainedFP.begin(),
                    PendingConstrainedFP.end());
PendingLoads.append(PendingConstrainedFPStrict.begin(),
                    PendingConstrainedFPStrict.end());
PendingConstrainedFP.clear();
PendingConstrainedFPStrict.clear();
return getMemoryRoot();
1124}

1126SDValue SelectionDAGBuilder::getControlRoot() {
// We need to emit pending fpexcept.strict constrained intrinsics,
// so append them to the PendingExports list.
PendingExports.append(PendingConstrainedFPStrict.begin(),
                      PendingConstrainedFPStrict.end());
PendingConstrainedFPStrict.clear();
return updateRoot(PendingExports);
1133}

1135void SelectionDAGBuilder::visit(const Instruction &I) {
// Set up outgoing PHI node register values before emitting the terminator.
if (I.isTerminator()) {
  HandlePHINodesInSuccessorBlocks(I.getParent());
}

// Add SDDbgValue nodes for any var locs here. Do so before updating
// SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
  // Add SDDbgValue nodes for any var locs here. Do so before updating
  // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
  for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
       It != End; ++It) {
    auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
    dropDanglingDebugInfo(Var, It->Expr);
    SmallVector<Value *> Values(It->Values.location_ops());
    if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
                          It->Values.hasArgList()))
      addDanglingDebugInfo(It, SDNodeOrder);
  }
}

// Increase the SDNodeOrder if dealing with a non-debug instruction.
if (!isa<DbgInfoIntrinsic>(I))
  ++SDNodeOrder;

CurInst = &I;

// Set inserted listener only if required.
bool NodeInserted = false;
std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
if (PCSectionsMD) {
  InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
      DAG, [&](SDNode *) { NodeInserted = true; });
}

visit(I.getOpcode(), I);

if (!I.isTerminator() && !HasTailCall &&
    !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
  CopyToExportRegsIfNeeded(&I);

// Handle metadata.
if (PCSectionsMD) {
  auto It = NodeMap.find(&I);
  if (It != NodeMap.end()) {
    DAG.addPCSections(It->second.getNode(), PCSectionsMD);
  } else if (NodeInserted) {
    // This should not happen; if it does, don't let it go unnoticed so we can
    // fix it. Relevant visit*() function is probably missing a setValue().
    errs() << "warning: loosing !pcsections metadata ["
           << I.getModule()->getName() << "]\n";
    LLVM_DEBUG(I.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { I.dump(); } } while (false);
    assert(false)(static_cast <bool> (false) ? void (0) : __assert_fail (
"false", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1189, __extension__ __PRETTY_FUNCTION__));
  }
}

CurInst = nullptr;
1194}

1196void SelectionDAGBuilder::visitPHI(const PHINode &) {
llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1197
);
1198}

1200void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
// Note: this doesn't use InstVisitor, because it has to work with
// ConstantExpr's in addition to instructions.
switch (Opcode) {
default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1204
);
  // Build the switch statement using the Instruction.def file.
1206#define HANDLE_INST(NUM, OPCODE, CLASS) \
  case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1208#include "llvm/IR/Instruction.def"
}
1210}

1212static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG,
                                          DILocalVariable *Variable,
                                          DebugLoc DL, unsigned Order,
                                          RawLocationWrapper Values,
                                          DIExpression *Expression) {
if (!Values.hasArgList())
  return false;
// For variadic dbg_values we will now insert an undef.
// FIXME: We can potentially recover these!
SmallVector<SDDbgOperand, 2> Locs;
for (const Value *V : Values.location_ops()) {
  auto *Undef = UndefValue::get(V->getType());
  Locs.push_back(SDDbgOperand::fromConst(Undef));
}
SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
                                      /*IsIndirect=*/false, DL, Order,
                                      /*IsVariadic=*/true);
DAG.AddDbgValue(SDV, /*isParameter=*/false);
return true;
1231}

1233void SelectionDAGBuilder::addDanglingDebugInfo(const VarLocInfo *VarLoc,
                                             unsigned Order) {
if (!handleDanglingVariadicDebugInfo(
        DAG,
        const_cast<DILocalVariable *>(DAG.getFunctionVarLocs()
                                          ->getVariable(VarLoc->VariableID)
                                          .getVariable()),
        VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) {
  DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back(
      VarLoc, Order);
}
1244}

1246void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI,
                                             unsigned Order) {
// We treat variadic dbg_values differently at this stage.
if (!handleDanglingVariadicDebugInfo(
        DAG, DI->getVariable(), DI->getDebugLoc(), Order,
        DI->getWrappedLocation(), DI->getExpression())) {
  // TODO: Dangling debug info will eventually either be resolved or produce
  // an Undef DBG_VALUE. However in the resolution case, a gap may appear
  // between the original dbg.value location and its resolved DBG_VALUE,
  // which we should ideally fill with an extra Undef DBG_VALUE.
  assert(DI->getNumVariableLocationOps() == 1 &&(static_cast <bool> (DI->getNumVariableLocationOps()
 == 1 && "DbgValueInst without an ArgList should have a single location "
 "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1258
, __extension__ __PRETTY_FUNCTION__))
         "DbgValueInst without an ArgList should have a single location "(static_cast <bool> (DI->getNumVariableLocationOps()
 == 1 && "DbgValueInst without an ArgList should have a single location "
 "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1258
, __extension__ __PRETTY_FUNCTION__))
         "operand.")(static_cast <bool> (DI->getNumVariableLocationOps()
 == 1 && "DbgValueInst without an ArgList should have a single location "
 "operand.") ? void (0) : __assert_fail ("DI->getNumVariableLocationOps() == 1 && \"DbgValueInst without an ArgList should have a single location \" \"operand.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1258
, __extension__ __PRETTY_FUNCTION__));
  DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order);
}
1261}

1263void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
                                              const DIExpression *Expr) {
auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
  DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs());
  DIExpression *DanglingExpr = DDI.getExpression();
  if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
    LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
 << printDDI(DDI) << "\n"; } } while (false)
                      << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
 << printDDI(DDI) << "\n"; } } while (false);
    return true;
  }
  return false;
};

for (auto &DDIMI : DanglingDebugInfoMap) {
  DanglingDebugInfoVector &DDIV = DDIMI.second;

  // If debug info is to be dropped, run it through final checks to see
  // whether it can be salvaged.
  for (auto &DDI : DDIV)
    if (isMatchingDbgValue(DDI))
      salvageUnresolvedDbgValue(DDI);

  erase_if(DDIV, isMatchingDbgValue);
}
1287}

1289// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1290// generate the debug data structures now that we've seen its definition.
1291void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
                                                 SDValue Val) {
auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
  return;

DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
for (auto &DDI : DDIV) {
  DebugLoc DL = DDI.getDebugLoc();
  unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
  unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
  DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs());
  DIExpression *Expr = DDI.getExpression();
  assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
 (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1305
, __extension__ __PRETTY_FUNCTION__))
         "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
 (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1305
, __extension__ __PRETTY_FUNCTION__));
  SDDbgValue *SDV;
  if (Val.getNode()) {
    // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
    // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
    // we couldn't resolve it directly when examining the DbgValue intrinsic
    // in the first place we should not be more successful here). Unless we
    // have some test case that prove this to be correct we should avoid
    // calling EmitFuncArgumentDbgValue here.
    if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
                                  FuncArgumentDbgValueKind::Value, Val)) {
      LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info for "
 << printDDI(DDI) << "\n"; } } while (false)
                        << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info for "
 << printDDI(DDI) << "\n"; } } while (false);
      LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  By mapping to:\n    "; Val.dump
(); } } while (false);
      // Increase the SDNodeOrder for the DbgValue here to make sure it is
      // inserted after the definition of Val when emitting the instructions
      // after ISel. An alternative could be to teach
      // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
      LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
 "changing SDNodeOrder from " << DbgSDNodeOrder <<
 " to " << ValSDNodeOrder << "\n"; } } while (false
)
                 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
 "changing SDNodeOrder from " << DbgSDNodeOrder <<
 " to " << ValSDNodeOrder << "\n"; } } while (false
)
                 << ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
 "changing SDNodeOrder from " << DbgSDNodeOrder <<
 " to " << ValSDNodeOrder << "\n"; } } while (false
);
      SDV = getDbgValue(Val, Variable, Expr, DL,
                        std::max(DbgSDNodeOrder, ValSDNodeOrder));
      DAG.AddDbgValue(SDV, false);
    } else
      LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
 << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n"
; } } while (false)
                        << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
 << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n"
; } } while (false);
  } else {
    LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
 printDDI(DDI) << "\n"; } } while (false);
    auto Undef = UndefValue::get(V->getType());
    auto SDV =
        DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
    DAG.AddDbgValue(SDV, false);
  }
}
DDIV.clear();
1341}

1343void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
// TODO: For the variadic implementation, instead of only checking the fail
// state of `handleDebugValue`, we need know specifically which values were
// invalid, so that we attempt to salvage only those values when processing
// a DIArgList.
Value *V = DDI.getVariableLocationOp(0);
Value *OrigV = V;
DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs());
DIExpression *Expr = DDI.getExpression();
DebugLoc DL = DDI.getDebugLoc();
unsigned SDOrder = DDI.getSDNodeOrder();

// Currently we consider only dbg.value intrinsics -- we tell the salvager
// that DW_OP_stack_value is desired.
bool StackValue = true;

// Can this Value can be encoded without any further work?
if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
  return;

// Attempt to salvage back through as many instructions as possible. Bail if
// a non-instruction is seen, such as a constant expression or global
// variable. FIXME: Further work could recover those too.
while (isa<Instruction>(V)) {
  Instruction &VAsInst = *cast<Instruction>(V);
  // Temporary "0", awaiting real implementation.
  SmallVector<uint64_t, 16> Ops;
  SmallVector<Value *, 4> AdditionalValues;
  V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops,
                           AdditionalValues);
  // If we cannot salvage any further, and haven't yet found a suitable debug
  // expression, bail out.
  if (!V)
    break;

  // TODO: If AdditionalValues isn't empty, then the salvage can only be
  // represented with a DBG_VALUE_LIST, so we give up. When we have support
  // here for variadic dbg_values, remove that condition.
  if (!AdditionalValues.empty())
    break;

  // New value and expr now represent this debuginfo.
  Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);

  // Some kind of simplification occurred: check whether the operand of the
  // salvaged debug expression can be encoded in this DAG.
  if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
    LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n  "
 << *Var << "\n" << *OrigV << "\nBy stripping back to:\n  "
 << *V << "\n"; } } while (false)
        dbgs() << "Salvaged debug location info for:\n  " << *Var << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n  "
 << *Var << "\n" << *OrigV << "\nBy stripping back to:\n  "
 << *V << "\n"; } } while (false)
               << *OrigV << "\nBy stripping back to:\n  " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Salvaged debug location info for:\n  "
 << *Var << "\n" << *OrigV << "\nBy stripping back to:\n  "
 << *V << "\n"; } } while (false);
    return;
  }
}

// This was the final opportunity to salvage this debug information, and it
// couldn't be done. Place an undef DBG_VALUE at this location to terminate
// any earlier variable location.
assert(OrigV && "V shouldn't be null")(static_cast <bool> (OrigV && "V shouldn't be null"
) ? void (0) : __assert_fail ("OrigV && \"V shouldn't be null\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1400
, __extension__ __PRETTY_FUNCTION__));
auto *Undef = UndefValue::get(OrigV->getType());
auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
DAG.AddDbgValue(SDV, false);
LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << printDDI(DDI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n  "
 << printDDI(DDI) << "\n"; } } while (false)
                  << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug value info for:\n  "
 << printDDI(DDI) << "\n"; } } while (false);
1406}

1408void SelectionDAGBuilder::handleKillDebugValue(DILocalVariable *Var,
                                             DIExpression *Expr,
                                             DebugLoc DbgLoc,
                                             unsigned Order) {
Value *Poison = PoisonValue::get(Type::getInt1Ty(*Context));
DIExpression *NewExpr =
    const_cast<DIExpression *>(DIExpression::convertToUndefExpression(Expr));
handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
                 /*IsVariadic*/ false);
1417}

1419bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values,
                                         DILocalVariable *Var,
                                         DIExpression *Expr, DebugLoc DbgLoc,
                                         unsigned Order, bool IsVariadic) {
if (Values.empty())
  return true;
SmallVector<SDDbgOperand> LocationOps;
SmallVector<SDNode *> Dependencies;
for (const Value *V : Values) {
  // Constant value.
  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
      isa<ConstantPointerNull>(V)) {
    LocationOps.emplace_back(SDDbgOperand::fromConst(V));
    continue;
  }

  // Look through IntToPtr constants.
  if (auto *CE = dyn_cast<ConstantExpr>(V))
    if (CE->getOpcode() == Instruction::IntToPtr) {
      LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
      continue;
    }

  // If the Value is a frame index, we can create a FrameIndex debug value
  // without relying on the DAG at all.
  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
    auto SI = FuncInfo.StaticAllocaMap.find(AI);
    if (SI != FuncInfo.StaticAllocaMap.end()) {
      LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
      continue;
    }
  }

  // Do not use getValue() in here; we don't want to generate code at
  // this point if it hasn't been done yet.
  SDValue N = NodeMap[V];
  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
    N = UnusedArgNodeMap[V];
  if (N.getNode()) {
    // Only emit func arg dbg value for non-variadic dbg.values for now.
    if (!IsVariadic &&
        EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
                                 FuncArgumentDbgValueKind::Value, N))
      return true;
    if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
      // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
      // describe stack slot locations.
      //
      // Consider "int x = 0; int *px = &x;". There are two kinds of
      // interesting debug values here after optimization:
      //
      //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
      //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
      //
      // Both describe the direct values of their associated variables.
      Dependencies.push_back(N.getNode());
      LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
      continue;
    }
    LocationOps.emplace_back(
        SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
    continue;
  }

  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  // Special rules apply for the first dbg.values of parameter variables in a
  // function. Identify them by the fact they reference Argument Values, that
  // they're parameters, and they are parameters of the current function. We
  // need to let them dangle until they get an SDNode.
  bool IsParamOfFunc =
      isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
  if (IsParamOfFunc)
    return false;

  // The value is not used in this block yet (or it would have an SDNode).
  // We still want the value to appear for the user if possible -- if it has
  // an associated VReg, we can refer to that instead.
  auto VMI = FuncInfo.ValueMap.find(V);
  if (VMI != FuncInfo.ValueMap.end()) {
    unsigned Reg = VMI->second;
    // If this is a PHI node, it may be split up into several MI PHI nodes
    // (in FunctionLoweringInfo::set).
    RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
                     V->getType(), std::nullopt);
    if (RFV.occupiesMultipleRegs()) {
      // FIXME: We could potentially support variadic dbg_values here.
      if (IsVariadic)
        return false;
      unsigned Offset = 0;
      unsigned BitsToDescribe = 0;
      if (auto VarSize = Var->getSizeInBits())
        BitsToDescribe = *VarSize;
      if (auto Fragment = Expr->getFragmentInfo())
        BitsToDescribe = Fragment->SizeInBits;
      for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
        // Bail out if all bits are described already.
        if (Offset >= BitsToDescribe)
          break;
        // TODO: handle scalable vectors.
        unsigned RegisterSize = RegAndSize.second;
        unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
                                    ? BitsToDescribe - Offset
                                    : RegisterSize;
        auto FragmentExpr = DIExpression::createFragmentExpression(
            Expr, Offset, FragmentSize);
        if (!FragmentExpr)
          continue;
        SDDbgValue *SDV = DAG.getVRegDbgValue(
            Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder);
        DAG.AddDbgValue(SDV, false);
        Offset += RegisterSize;
      }
      return true;
    }
    // We can use simple vreg locations for variadic dbg_values as well.
    LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
    continue;
  }
  // We failed to create a SDDbgOperand for V.
  return false;
}

// We have created a SDDbgOperand for each Value in Values.
// Should use Order instead of SDNodeOrder?
assert(!LocationOps.empty())(static_cast <bool> (!LocationOps.empty()) ? void (0) :
 __assert_fail ("!LocationOps.empty()", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1543, __extension__ __PRETTY_FUNCTION__));
SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
                                      /*IsIndirect=*/false, DbgLoc,
                                      SDNodeOrder, IsVariadic);
DAG.AddDbgValue(SDV, /*isParameter=*/false);
return true;
1549}

1551void SelectionDAGBuilder::resolveOrClearDbgInfo() {
// Try to fixup any remaining dangling debug info -- and drop it if we can't.
for (auto &Pair : DanglingDebugInfoMap)
  for (auto &DDI : Pair.second)
    salvageUnresolvedDbgValue(DDI);
clearDanglingDebugInfo();
1557}

1559/// getCopyFromRegs - If there was virtual register allocated for the value V
1560/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1561SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V);
SDValue Result;

if (It != FuncInfo.ValueMap.end()) {
  Register InReg = It->second;

  RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
                   DAG.getDataLayout(), InReg, Ty,
                   std::nullopt); // This is not an ABI copy.
  SDValue Chain = DAG.getEntryNode();
  Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
                               V);
  resolveDanglingDebugInfo(V, Result);
}

return Result;
1578}

1580/// getValue - Return an SDValue for the given Value.
1581SDValue SelectionDAGBuilder::getValue(const Value *V) {
// If we already have an SDValue for this value, use it. It's important
// to do this first, so that we don't create a CopyFromReg if we already
// have a regular SDValue.
SDValue &N = NodeMap[V];
if (N.getNode()) return N;

// If there's a virtual register allocated and initialized for this
// value, use it.
if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
  return copyFromReg;

// Otherwise create a new SDValue and remember it.
SDValue Val = getValueImpl(V);
NodeMap[V] = Val;
resolveDanglingDebugInfo(V, Val);
return Val;
1598}

1600/// getNonRegisterValue - Return an SDValue for the given Value, but
1601/// don't look in FuncInfo.ValueMap for a virtual register.
1602SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
// If we already have an SDValue for this value, use it.
SDValue &N = NodeMap[V];
if (N.getNode()) {
  if (isIntOrFPConstant(N)) {
    // Remove the debug location from the node as the node is about to be used
    // in a location which may differ from the original debug location.  This
    // is relevant to Constant and ConstantFP nodes because they can appear
    // as constant expressions inside PHI nodes.
    N->setDebugLoc(DebugLoc());
  }
  return N;
}

// Otherwise create a new SDValue and remember it.
SDValue Val = getValueImpl(V);
NodeMap[V] = Val;
resolveDanglingDebugInfo(V, Val);
return Val;
1621}

1623/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1624/// Create an SDValue for the given value.
1625SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

if (const Constant *C = dyn_cast<Constant>(V)) {
  EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);

  if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
    return DAG.getConstant(*CI, getCurSDLoc(), VT);

  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
    return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);

  if (isa<ConstantPointerNull>(C)) {
    unsigned AS = V->getType()->getPointerAddressSpace();
    return DAG.getConstant(0, getCurSDLoc(),
                           TLI.getPointerTy(DAG.getDataLayout(), AS));
  }

  if (match(C, m_VScale()))
    return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));

  if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
    return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);

  if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
    return DAG.getUNDEF(VT);

  if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
    visit(CE->getOpcode(), *CE);
    SDValue N1 = NodeMap[V];
    assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!"
) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1655
, __extension__ __PRETTY_FUNCTION__));
    return N1;
  }

  if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
    SmallVector<SDValue, 4> Constants;
    for (const Use &U : C->operands()) {
      SDNode *Val = getValue(U).getNode();
      // If the operand is an empty aggregate, there are no values.
      if (!Val) continue;
      // Add each leaf value from the operand to the Constants list
      // to form a flattened list of all the values.
      for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
        Constants.push_back(SDValue(Val, i));
    }

    return DAG.getMergeValues(Constants, getCurSDLoc());
  }

  if (const ConstantDataSequential *CDS =
        dyn_cast<ConstantDataSequential>(C)) {
    SmallVector<SDValue, 4> Ops;
    for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
      SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
      // Add each leaf value from the operand to the Constants list
      // to form a flattened list of all the values.
      for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
        Ops.push_back(SDValue(Val, i));
    }

    if (isa<ArrayType>(CDS->getType()))
      return DAG.getMergeValues(Ops, getCurSDLoc());
    return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
  }

  if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
    assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1692
, __extension__ __PRETTY_FUNCTION__))
           "Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1692
, __extension__ __PRETTY_FUNCTION__));

    SmallVector<EVT, 4> ValueVTs;
    ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
    unsigned NumElts = ValueVTs.size();
    if (NumElts == 0)
      return SDValue(); // empty struct
    SmallVector<SDValue, 4> Constants(NumElts);
    for (unsigned i = 0; i != NumElts; ++i) {
      EVT EltVT = ValueVTs[i];
      if (isa<UndefValue>(C))
        Constants[i] = DAG.getUNDEF(EltVT);
      else if (EltVT.isFloatingPoint())
        Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
      else
        Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
    }

    return DAG.getMergeValues(Constants, getCurSDLoc());
  }

  if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
    return DAG.getBlockAddress(BA, VT);

  if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
    return getValue(Equiv->getGlobalValue());

  if (const auto *NC = dyn_cast<NoCFIValue>(C))
    return getValue(NC->getGlobalValue());

  VectorType *VecTy = cast<VectorType>(V->getType());

  // Now that we know the number and type of the elements, get that number of
  // elements into the Ops array based on what kind of constant it is.
  if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
    SmallVector<SDValue, 16> Ops;
    unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
    for (unsigned i = 0; i != NumElements; ++i)
      Ops.push_back(getValue(CV->getOperand(i)));

    return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
  }

  if (isa<ConstantAggregateZero>(C)) {
    EVT EltVT =
        TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());

    SDValue Op;
    if (EltVT.isFloatingPoint())
      Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
    else
      Op = DAG.getConstant(0, getCurSDLoc(), EltVT);

    return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
  }

  llvm_unreachable("Unknown vector constant")::llvm::llvm_unreachable_internal("Unknown vector constant", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1748);
}

// If this is a static alloca, generate it as the frameindex instead of
// computation.
if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
  DenseMap<const AllocaInst*, int>::iterator SI =
    FuncInfo.StaticAllocaMap.find(AI);
  if (SI != FuncInfo.StaticAllocaMap.end())
    return DAG.getFrameIndex(
        SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
}

// If this is an instruction which fast-isel has deferred, select it now.
if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
  Register InReg = FuncInfo.InitializeRegForValue(Inst);

  RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
                   Inst->getType(), std::nullopt);
  SDValue Chain = DAG.getEntryNode();
  return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
}

if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
  return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));

if (const auto *BB = dyn_cast<BasicBlock>(V))
  return DAG.getBasicBlock(FuncInfo.MBBMap[BB]);

llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1777
);
1778}

1780void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
bool IsSEH = isAsynchronousEHPersonality(Pers);
MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
if (!IsSEH)
  CatchPadMBB->setIsEHScopeEntry();
// In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
if (IsMSVCCXX || IsCoreCLR)
  CatchPadMBB->setIsEHFuncletEntry();
1791}

1793void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
// Update machine-CFG edge.
MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
FuncInfo.MBB->addSuccessor(TargetMBB);
TargetMBB->setIsEHCatchretTarget(true);
DAG.getMachineFunction().setHasEHCatchret(true);

auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
bool IsSEH = isAsynchronousEHPersonality(Pers);
if (IsSEH) {
  // If this is not a fall-through branch or optimizations are switched off,
  // emit the branch.
  if (TargetMBB != NextBlock(FuncInfo.MBB) ||
      TM.getOptLevel() == CodeGenOpt::None)
    DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
                            getControlRoot(), DAG.getBasicBlock(TargetMBB)));
  return;
}

// Figure out the funclet membership for the catchret's successor.
// This will be used by the FuncletLayout pass to determine how to order the
// BB's.
// A 'catchret' returns to the outer scope's color.
Value *ParentPad = I.getCatchSwitchParentPad();
const BasicBlock *SuccessorColor;
if (isa<ConstantTokenNone>(ParentPad))
  SuccessorColor = &FuncInfo.Fn->getEntryBlock();
else
  SuccessorColor = cast<Instruction>(ParentPad)->getParent();
assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!"
) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1822
, __extension__ __PRETTY_FUNCTION__));
MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!"
) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1824
, __extension__ __PRETTY_FUNCTION__));

// Create the terminator node.
SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
                          getControlRoot(), DAG.getBasicBlock(TargetMBB),
                          DAG.getBasicBlock(SuccessorColorMBB));
DAG.setRoot(Ret);
1831}

1833void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
// Don't emit any special code for the cleanuppad instruction. It just marks
// the start of an EH scope/funclet.
FuncInfo.MBB->setIsEHScopeEntry();
auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
if (Pers != EHPersonality::Wasm_CXX) {
  FuncInfo.MBB->setIsEHFuncletEntry();
  FuncInfo.MBB->setIsCleanupFuncletEntry();
}
1842}

1844// In wasm EH, even though a catchpad may not catch an exception if a tag does
1845// not match, it is OK to add only the first unwind destination catchpad to the
1846// successors, because there will be at least one invoke instruction within the
1847// catch scope that points to the next unwind destination, if one exists, so
1848// CFGSort cannot mess up with BB sorting order.
1849// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
1850// call within them, and catchpads only consisting of 'catch (...)' have a
1851// '__cxa_end_catch' call within them, both of which generate invokes in case
1852// the next unwind destination exists, i.e., the next unwind destination is not
1853// the caller.)
1854//
1855// Having at most one EH pad successor is also simpler and helps later
1856// transformations.
1857//
1858// For example,
1859// current:
1860//   invoke void @foo to ... unwind label %catch.dispatch
1861// catch.dispatch:
1862//   %0 = catchswitch within ... [label %catch.start] unwind label %next
1863// catch.start:
1864//   ...
1865//   ... in this BB or some other child BB dominated by this BB there will be an
1866//   invoke that points to 'next' BB as an unwind destination
1867//
1868// next: ; We don't need to add this to 'current' BB's successor
1869//   ...
1870static void findWasmUnwindDestinations(
  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
  BranchProbability Prob,
  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
      &UnwindDests) {
while (EHPadBB) {
  const Instruction *Pad = EHPadBB->getFirstNonPHI();
  if (isa<CleanupPadInst>(Pad)) {
    // Stop on cleanup pads.
    UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
    UnwindDests.back().first->setIsEHScopeEntry();
    break;
  } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
    // Add the catchpad handlers to the possible destinations. We don't
    // continue to the unwind destination of the catchswitch for wasm.
    for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
      UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
      UnwindDests.back().first->setIsEHScopeEntry();
    }
    break;
  } else {
    continue;
  }
}
1894}

1896/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1897/// many places it could ultimately go. In the IR, we have a single unwind
1898/// destination, but in the machine CFG, we enumerate all the possible blocks.
1899/// This function skips over imaginary basic blocks that hold catchswitch
1900/// instructions, and finds all the "real" machine
1901/// basic block destinations. As those destinations may not be successors of
1902/// EHPadBB, here we also calculate the edge probability to those destinations.
1903/// The passed-in Prob is the edge probability to EHPadBB.
1904static void findUnwindDestinations(
  FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
  BranchProbability Prob,
  SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
      &UnwindDests) {
EHPersonality Personality =
  classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
bool IsSEH = isAsynchronousEHPersonality(Personality);

if (IsWasmCXX) {
  findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
  assert(UnwindDests.size() <= 1 &&(static_cast <bool> (UnwindDests.size() <= 1 &&
 "There should be at most one unwind destination for wasm") ?
 void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1919
, __extension__ __PRETTY_FUNCTION__))
         "There should be at most one unwind destination for wasm")(static_cast <bool> (UnwindDests.size() <= 1 &&
 "There should be at most one unwind destination for wasm") ?
 void (0) : __assert_fail ("UnwindDests.size() <= 1 && \"There should be at most one unwind destination for wasm\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 1919
, __extension__ __PRETTY_FUNCTION__));
  return;
}

while (EHPadBB) {
  const Instruction *Pad = EHPadBB->getFirstNonPHI();
  BasicBlock *NewEHPadBB = nullptr;
  if (isa<LandingPadInst>(Pad)) {
    // Stop on landingpads. They are not funclets.
    UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
    break;
  } else if (isa<CleanupPadInst>(Pad)) {
    // Stop on cleanup pads. Cleanups are always funclet entries for all known
    // personalities.
    UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
    UnwindDests.back().first->setIsEHScopeEntry();
    UnwindDests.back().first->setIsEHFuncletEntry();
    break;
  } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
    // Add the catchpad handlers to the possible destinations.
    for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
      UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
      // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
      if (IsMSVCCXX || IsCoreCLR)
        UnwindDests.back().first->setIsEHFuncletEntry();
      if (!IsSEH)
        UnwindDests.back().first->setIsEHScopeEntry();
    }
    NewEHPadBB = CatchSwitch->getUnwindDest();
  } else {
    continue;
  }

  BranchProbabilityInfo *BPI = FuncInfo.BPI;
  if (BPI && NewEHPadBB)
    Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
  EHPadBB = NewEHPadBB;
}
1957}

1959void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
// Update successor info.
SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
auto UnwindDest = I.getUnwindDest();
BranchProbabilityInfo *BPI = FuncInfo.BPI;
BranchProbability UnwindDestProb =
    (BPI && UnwindDest)
        ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
        : BranchProbability::getZero();
findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
for (auto &UnwindDest : UnwindDests) {
  UnwindDest.first->setIsEHPad();
  addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
}
FuncInfo.MBB->normalizeSuccProbs();

// Create the terminator node.
SDValue Ret =
    DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
DAG.setRoot(Ret);
1979}

1981void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
report_fatal_error("visitCatchSwitch not yet implemented!");
1983}

1985void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto &DL = DAG.getDataLayout();
SDValue Chain = getControlRoot();
SmallVector<ISD::OutputArg, 8> Outs;
SmallVector<SDValue, 8> OutVals;

// Calls to @llvm.experimental.deoptimize don't generate a return value, so
// lower
//
//   %val = call <ty> @llvm.experimental.deoptimize()
//   ret <ty> %val
//
// differently.
if (I.getParent()->getTerminatingDeoptimizeCall()) {
  LowerDeoptimizingReturn();
  return;
}

if (!FuncInfo.CanLowerReturn) {
  unsigned DemoteReg = FuncInfo.DemoteRegister;
  const Function *F = I.getParent()->getParent();

  // Emit a store of the return value through the virtual register.
  // Leave Outs empty so that LowerReturn won't try to load return
  // registers the usual way.
  SmallVector<EVT, 1> PtrValueVTs;
  ComputeValueVTs(TLI, DL,
                  F->getReturnType()->getPointerTo(
                      DAG.getDataLayout().getAllocaAddrSpace()),
                  PtrValueVTs);

  SDValue RetPtr =
      DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]);
  SDValue RetOp = getValue(I.getOperand(0));

  SmallVector<EVT, 4> ValueVTs, MemVTs;
  SmallVector<uint64_t, 4> Offsets;
  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
                  &Offsets);
  unsigned NumValues = ValueVTs.size();

  SmallVector<SDValue, 4> Chains(NumValues);
  Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
  for (unsigned i = 0; i != NumValues; ++i) {
    // An aggregate return value cannot wrap around the address space, so
    // offsets to its parts don't wrap either.
    SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr,
                                         TypeSize::Fixed(Offsets[i]));

    SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
    if (MemVTs[i] != ValueVTs[i])
      Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
    Chains[i] = DAG.getStore(
        Chain, getCurSDLoc(), Val,
        // FIXME: better loc info would be nice.
        Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
        commonAlignment(BaseAlign, Offsets[i]));
  }

  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
                      MVT::Other, Chains);
} else if (I.getNumOperands() != 0) {
  SmallVector<EVT, 4> ValueVTs;
  ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
  unsigned NumValues = ValueVTs.size();
  if (NumValues) {
    SDValue RetOp = getValue(I.getOperand(0));

    const Function *F = I.getParent()->getParent();

    bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
        I.getOperand(0)->getType(), F->getCallingConv(),
        /*IsVarArg*/ false, DL);

    ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
    if (F->getAttributes().hasRetAttr(Attribute::SExt))
      ExtendKind = ISD::SIGN_EXTEND;
    else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
      ExtendKind = ISD::ZERO_EXTEND;

    LLVMContext &Context = F->getContext();
    bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);

    for (unsigned j = 0; j != NumValues; ++j) {
      EVT VT = ValueVTs[j];

      if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
        VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);

      CallingConv::ID CC = F->getCallingConv();

      unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
      MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
      SmallVector<SDValue, 4> Parts(NumParts);
      getCopyToParts(DAG, getCurSDLoc(),
                     SDValue(RetOp.getNode(), RetOp.getResNo() + j),
                     &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);

      // 'inreg' on function refers to return value
      ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
      if (RetInReg)
        Flags.setInReg();

      if (I.getOperand(0)->getType()->isPointerTy()) {
        Flags.setPointer();
        Flags.setPointerAddrSpace(
            cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
      }

      if (NeedsRegBlock) {
        Flags.setInConsecutiveRegs();
        if (j == NumValues - 1)
          Flags.setInConsecutiveRegsLast();
      }

      // Propagate extension type if any
      if (ExtendKind == ISD::SIGN_EXTEND)
        Flags.setSExt();
      else if (ExtendKind == ISD::ZERO_EXTEND)
        Flags.setZExt();

      for (unsigned i = 0; i < NumParts; ++i) {
        Outs.push_back(ISD::OutputArg(Flags,
                                      Parts[i].getValueType().getSimpleVT(),
                                      VT, /*isfixed=*/true, 0, 0));
        OutVals.push_back(Parts[i]);
      }
    }
  }
}

// Push in swifterror virtual register as the last element of Outs. This makes
// sure swifterror virtual register will be returned in the swifterror
// physical register.
const Function *F = I.getParent()->getParent();
if (TLI.supportSwiftError() &&
    F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
  assert(SwiftError.getFunctionArg() && "Need a swift error argument")(static_cast <bool> (SwiftError.getFunctionArg() &&
 "Need a swift error argument") ? void (0) : __assert_fail ("SwiftError.getFunctionArg() && \"Need a swift error argument\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2123
, __extension__ __PRETTY_FUNCTION__));
  ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
  Flags.setSwiftError();
  Outs.push_back(ISD::OutputArg(
      Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
      /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
  // Create SDNode for the swifterror virtual register.
  OutVals.push_back(
      DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
                          &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
                      EVT(TLI.getPointerTy(DL))));
}

bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
CallingConv::ID CallConv =
  DAG.getMachineFunction().getFunction().getCallingConv();
Chain = DAG.getTargetLoweringInfo().LowerReturn(
    Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);

// Verify that the target's LowerReturn behaved as expected.
assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2144
, __extension__ __PRETTY_FUNCTION__))
       "LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2144
, __extension__ __PRETTY_FUNCTION__));

// Update the DAG with the new chain value resulting from return lowering.
DAG.setRoot(Chain);
2148}

2150/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2151/// created for it, emit nodes to copy the value into the virtual
2152/// registers.
2153void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
// Skip empty types
if (V->getType()->isEmptyTy())
  return;

DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V);
if (VMI != FuncInfo.ValueMap.end()) {
  assert((!V->use_empty() || isa<CallBrInst>(V)) &&(static_cast <bool> ((!V->use_empty() || isa<CallBrInst
>(V)) && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("(!V->use_empty() || isa<CallBrInst>(V)) && \"Unused value assigned virtual registers!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2161
, __extension__ __PRETTY_FUNCTION__))
         "Unused value assigned virtual registers!")(static_cast <bool> ((!V->use_empty() || isa<CallBrInst
>(V)) && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("(!V->use_empty() || isa<CallBrInst>(V)) && \"Unused value assigned virtual registers!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2161
, __extension__ __PRETTY_FUNCTION__));
  CopyValueToVirtualRegister(V, VMI->second);
}
2164}

2166/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2167/// the current basic block, add it to ValueMap now so that we'll get a
2168/// CopyTo/FromReg.
2169void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
// No need to export constants.
if (!isa<Instruction>(V) && !isa<Argument>(V)) return;

// Already exported?
if (FuncInfo.isExportedInst(V)) return;

Register Reg = FuncInfo.InitializeRegForValue(V);
CopyValueToVirtualRegister(V, Reg);
2178}

2180bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
                                                   const BasicBlock *FromBB) {
// The operands of the setcc have to be in this block.  We don't know
// how to export them from some other block.
if (const Instruction *VI = dyn_cast<Instruction>(V)) {
  // Can export from current BB.
  if (VI->getParent() == FromBB)
    return true;

  // Is already exported, noop.
  return FuncInfo.isExportedInst(V);
}

// If this is an argument, we can export it if the BB is the entry block or
// if it is already exported.
if (isa<Argument>(V)) {
  if (FromBB->isEntryBlock())
    return true;

  // Otherwise, can only export this if it is already exported.
  return FuncInfo.isExportedInst(V);
}

// Otherwise, constants can always be exported.
return true;
2205}

2207/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2208BranchProbability
2209SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
                                      const MachineBasicBlock *Dst) const {
BranchProbabilityInfo *BPI = FuncInfo.BPI;
const BasicBlock *SrcBB = Src->getBasicBlock();
const BasicBlock *DstBB = Dst->getBasicBlock();
if (!BPI) {
  // If BPI is not available, set the default probability as 1 / N, where N is
  // the number of successors.
  auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
  return BranchProbability(1, SuccSize);
}
return BPI->getEdgeProbability(SrcBB, DstBB);
2221}

2223void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
                                             MachineBasicBlock *Dst,
                                             BranchProbability Prob) {
if (!FuncInfo.BPI)
  Src->addSuccessorWithoutProb(Dst);
else {
  if (Prob.isUnknown())
    Prob = getEdgeProbability(Src, Dst);
  Src->addSuccessor(Dst, Prob);
}
2233}

2235static bool InBlock(const Value *V, const BasicBlock *BB) {
if (const Instruction *I = dyn_cast<Instruction>(V))
  return I->getParent() == BB;
return true;
2239}

2241/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2242/// This function emits a branch and is used at the leaves of an OR or an
2243/// AND operator tree.
2244void
2245SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
                                                MachineBasicBlock *TBB,
                                                MachineBasicBlock *FBB,
                                                MachineBasicBlock *CurBB,
                                                MachineBasicBlock *SwitchBB,
                                                BranchProbability TProb,
                                                BranchProbability FProb,
                                                bool InvertCond) {
const BasicBlock *BB = CurBB->getBasicBlock();

// If the leaf of the tree is a comparison, merge the condition into
// the caseblock.
if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
  // The operands of the cmp have to be in this block.  We don't know
  // how to export them from some other block.  If this is the first block
  // of the sequence, no exporting is needed.
  if (CurBB == SwitchBB ||
      (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
       isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
    ISD::CondCode Condition;
    if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
      ICmpInst::Predicate Pred =
          InvertCond ? IC->getInversePredicate() : IC->getPredicate();
      Condition = getICmpCondCode(Pred);
    } else {
      const FCmpInst *FC = cast<FCmpInst>(Cond);
      FCmpInst::Predicate Pred =
          InvertCond ? FC->getInversePredicate() : FC->getPredicate();
      Condition = getFCmpCondCode(Pred);
      if (TM.Options.NoNaNsFPMath)
        Condition = getFCmpCodeWithoutNaN(Condition);
    }

    CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
                 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
    SL->SwitchCases.push_back(CB);
    return;
  }
}

// Create a CaseBlock record representing this branch.
ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
             nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
SL->SwitchCases.push_back(CB);
2290}

2292void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
                                             MachineBasicBlock *TBB,
                                             MachineBasicBlock *FBB,
                                             MachineBasicBlock *CurBB,
                                             MachineBasicBlock *SwitchBB,
                                             Instruction::BinaryOps Opc,
                                             BranchProbability TProb,
                                             BranchProbability FProb,
                                             bool InvertCond) {
// Skip over not part of the tree and remember to invert op and operands at
// next level.
Value *NotCond;
if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
    InBlock(NotCond, CurBB->getBasicBlock())) {
  FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
                       !InvertCond);
  return;
}

const Instruction *BOp = dyn_cast<Instruction>(Cond);
const Value *BOpOp0, *BOpOp1;
// Compute the effective opcode for Cond, taking into account whether it needs
// to be inverted, e.g.
//   and (not (or A, B)), C
// gets lowered as
//   and (and (not A, not B), C)
Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;
if (BOp) {
  BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
             ? Instruction::And
             : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
                    ? Instruction::Or
                    : (Instruction::BinaryOps)0);
  if (InvertCond) {
    if (BOpc == Instruction::And)
      BOpc = Instruction::Or;
    else if (BOpc == Instruction::Or)
      BOpc = Instruction::And;
  }
}

// If this node is not part of the or/and tree, emit it as a branch.
// Note that all nodes in the tree should have same opcode.
bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
    !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
    !InBlock(BOpOp1, CurBB->getBasicBlock())) {
  EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
                               TProb, FProb, InvertCond);
  return;
}

//  Create TmpBB after CurBB.
MachineFunction::iterator BBI(CurBB);
MachineFunction &MF = DAG.getMachineFunction();
MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
CurBB->getParent()->insert(++BBI, TmpBB);

if (Opc == Instruction::Or) {
  // Codegen X | Y as:
  // BB1:
  //   jmp_if_X TBB
  //   jmp TmpBB
  // TmpBB:
  //   jmp_if_Y TBB
  //   jmp FBB
  //

  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
  // The requirement is that
  //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
  //     = TrueProb for original BB.
  // Assuming the original probabilities are A and B, one choice is to set
  // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
  // A/(1+B) and 2B/(1+B). This choice assumes that
  //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
  // Another choice is to assume TrueProb for BB1 equals to TrueProb for
  // TmpBB, but the math is more complicated.

  auto NewTrueProb = TProb / 2;
  auto NewFalseProb = TProb / 2 + FProb;
  // Emit the LHS condition.
  FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
                       NewFalseProb, InvertCond);

  // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
  SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
  // Emit the RHS condition into TmpBB.
  FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
                       Probs[1], InvertCond);
} else {
  assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And &&
 "Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2384
, __extension__ __PRETTY_FUNCTION__));
  // Codegen X & Y as:
  // BB1:
  //   jmp_if_X TmpBB
  //   jmp FBB
  // TmpBB:
  //   jmp_if_Y TBB
  //   jmp FBB
  //
  //  This requires creation of TmpBB after CurBB.

  // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
  // The requirement is that
  //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
  //     = FalseProb for original BB.
  // Assuming the original probabilities are A and B, one choice is to set
  // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
  // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
  // TrueProb for BB1 * FalseProb for TmpBB.

  auto NewTrueProb = TProb + FProb / 2;
  auto NewFalseProb = FProb / 2;
  // Emit the LHS condition.
  FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
                       NewFalseProb, InvertCond);

  // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
  SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
  BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
  // Emit the RHS condition into TmpBB.
  FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
                       Probs[1], InvertCond);
}
2417}

2419/// If the set of cases should be emitted as a series of branches, return true.
2420/// If we should emit this as a bunch of and/or'd together conditions, return
2421/// false.
2422bool
2423SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
if (Cases.size() != 2) return true;

// If this is two comparisons of the same values or'd or and'd together, they
// will get folded into a single comparison, so don't emit two blocks.
if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
     Cases[0].CmpRHS == Cases[1].CmpRHS) ||
    (Cases[0].CmpRHS == Cases[1].CmpLHS &&
     Cases[0].CmpLHS == Cases[1].CmpRHS)) {
  return false;
}

// Handle: (X != null) | (Y != null) --> (X|Y) != 0
// Handle: (X == null) & (Y == null) --> (X|Y) == 0
if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
    Cases[0].CC == Cases[1].CC &&
    isa<Constant>(Cases[0].CmpRHS) &&
    cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
  if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
    return false;
  if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
    return false;
}

return true;
2448}

2450void SelectionDAGBuilder::visitBr(const BranchInst &I) {
MachineBasicBlock *BrMBB = FuncInfo.MBB;

// Update machine-CFG edges.
MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];

if (I.isUnconditional()) {
  // Update machine-CFG edges.
  BrMBB->addSuccessor(Succ0MBB);

  // If this is not a fall-through branch or optimizations are switched off,
  // emit the branch.
  if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
    DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
                            MVT::Other, getControlRoot(),
                            DAG.getBasicBlock(Succ0MBB)));

  return;
}

// If this condition is one of the special cases we handle, do special stuff
// now.
const Value *CondVal = I.getCondition();
MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];

// If this is a series of conditions that are or'd or and'd together, emit
// this as a sequence of branches instead of setcc's with and/or operations.
// As long as jumps are not expensive (exceptions for multi-use logic ops,
// unpredictable branches, and vector extracts because those jumps are likely
// expensive for any target), this should improve performance.
// For example, instead of something like:
//     cmp A, B
//     C = seteq
//     cmp D, E
//     F = setle
//     or C, F
//     jnz foo
// Emit:
//     cmp A, B
//     je foo
//     cmp D, E
//     jle foo
const Instruction *BOp = dyn_cast<Instruction>(CondVal);
if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
    BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
  Value *Vec;
  const Value *BOp0, *BOp1;
  Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
  if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
    Opcode = Instruction::And;
  else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
    Opcode = Instruction::Or;

  if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
                  match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {
    FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
                         getEdgeProbability(BrMBB, Succ0MBB),
                         getEdgeProbability(BrMBB, Succ1MBB),
                         /*InvertCond=*/false);
    // If the compares in later blocks need to use values not currently
    // exported from this block, export them now.  This block should always
    // be the first entry.
    assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SL->SwitchCases[0].ThisBB == BrMBB
 && "Unexpected lowering!") ? void (0) : __assert_fail
 ("SL->SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2512
, __extension__ __PRETTY_FUNCTION__));

    // Allow some cases to be rejected.
    if (ShouldEmitAsBranches(SL->SwitchCases)) {
      for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
        ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
        ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
      }

      // Emit the branch for this block.
      visitSwitchCase(SL->SwitchCases[0], BrMBB);
      SL->SwitchCases.erase(SL->SwitchCases.begin());
      return;
    }

    // Okay, we decided not to do this, remove any inserted MBB's and clear
    // SwitchCases.
    for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
      FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);

    SL->SwitchCases.clear();
  }
}

// Create a CaseBlock record representing this branch.
CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
             nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());

// Use visitSwitchCase to actually insert the fast branch sequence for this
// cond branch.
visitSwitchCase(CB, BrMBB);
2543}

2545/// visitSwitchCase - Emits the necessary code to represent a single node in
2546/// the binary search tree resulting from lowering a switch instruction.
2547void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
                                        MachineBasicBlock *SwitchBB) {
SDValue Cond;
SDValue CondLHS = getValue(CB.CmpLHS);
SDLoc dl = CB.DL;

if (CB.CC == ISD::SETTRUE) {
  // Branch or fall through to TrueBB.
  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
  SwitchBB->normalizeSuccProbs();
  if (CB.TrueBB != NextBlock(SwitchBB)) {
    DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
                            DAG.getBasicBlock(CB.TrueBB)));
  }
  return;
}

auto &TLI = DAG.getTargetLoweringInfo();
EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());

// Build the setcc now.
if (!CB.CmpMHS) {
  // Fold "(X == true)" to X and "(X == false)" to !X to
  // handle common cases produced by branch lowering.
  if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
      CB.CC == ISD::SETEQ)
    Cond = CondLHS;
  else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
           CB.CC == ISD::SETEQ) {
    SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
    Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
  } else {
    SDValue CondRHS = getValue(CB.CmpRHS);

    // If a pointer's DAG type is larger than its memory type then the DAG
    // values are zero-extended. This breaks signed comparisons so truncate
    // back to the underlying type before doing the compare.
    if (CondLHS.getValueType() != MemVT) {
      CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
      CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
    }
    Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
  }
} else {
  assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2591
, __extension__ __PRETTY_FUNCTION__));

  const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
  const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();

  SDValue CmpOp = getValue(CB.CmpMHS);
  EVT VT = CmpOp.getValueType();

  if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
    Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
                        ISD::SETLE);
  } else {
    SDValue SUB = DAG.getNode(ISD::SUB, dl,
                              VT, CmpOp, DAG.getConstant(Low, dl, VT));
    Cond = DAG.getSetCC(dl, MVT::i1, SUB,
                        DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
  }
}

// Update successor info
addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
// TrueBB and FalseBB are always different unless the incoming IR is
// degenerate. This only happens when running llc on weird IR.
if (CB.TrueBB != CB.FalseBB)
  addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
SwitchBB->normalizeSuccProbs();

// If the lhs block is the next block, invert the condition so that we can
// fall through to the lhs instead of the rhs block.
if (CB.TrueBB == NextBlock(SwitchBB)) {
  std::swap(CB.TrueBB, CB.FalseBB);
  SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
  Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
}

SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
                             MVT::Other, getControlRoot(), Cond,
                             DAG.getBasicBlock(CB.TrueBB));

setValue(CurInst, BrCond);

// Insert the false branch. Do this even if it's a fall through branch,
// this makes it easier to do DAG optimizations which require inverting
// the branch condition.
BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
                     DAG.getBasicBlock(CB.FalseBB));

DAG.setRoot(BrCond);
2639}

2641/// visitJumpTable - Emit JumpTable node in the current MBB
2642void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
// Emit the code for the jump table
assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!"
) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2644
, __extension__ __PRETTY_FUNCTION__));
EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
                                   JT.Reg, PTy);
SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
                                  MVT::Other, Index.getValue(1),
                                  Table, Index);
DAG.setRoot(BrJumpTable);
2653}

2655/// visitJumpTableHeader - This function emits necessary code to produce index
2656/// in the JumpTable from switch case.
2657void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
                                             JumpTableHeader &JTH,
                                             MachineBasicBlock *SwitchBB) {
SDLoc dl = getCurSDLoc();

// Subtract the lowest switch case value from the value being switched on.
SDValue SwitchOp = getValue(JTH.SValue);
EVT VT = SwitchOp.getValueType();
SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
                          DAG.getConstant(JTH.First, dl, VT));

// The SDNode we just created, which holds the value being switched on minus
// the smallest case value, needs to be copied to a virtual register so it
// can be used as an index into the jump table in a subsequent basic block.
// This value may be smaller or larger than the target's pointer type, and
// therefore require extension or truncating.
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));

unsigned JumpTableReg =
    FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
                                  JumpTableReg, SwitchOp);
JT.Reg = JumpTableReg;

if (!JTH.FallthroughUnreachable) {
  // Emit the range check for the jump table, and branch to the default block
  // for the switch statement if the value being switched on exceeds the
  // largest case in the switch.
  SDValue CMP = DAG.getSetCC(
      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
                                 Sub.getValueType()),
      Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);

  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
                               MVT::Other, CopyTo, CMP,
                               DAG.getBasicBlock(JT.Default));

  // Avoid emitting unnecessary branches to the next block.
  if (JT.MBB != NextBlock(SwitchBB))
    BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
                         DAG.getBasicBlock(JT.MBB));

  DAG.setRoot(BrCond);
} else {
  // Avoid emitting unnecessary branches to the next block.
  if (JT.MBB != NextBlock(SwitchBB))
    DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
                            DAG.getBasicBlock(JT.MBB)));
  else
    DAG.setRoot(CopyTo);
}
2709}

2711/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2712/// variable if there exists one.
2713static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
                               SDValue &Chain) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
MachineFunction &MF = DAG.getMachineFunction();
Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
MachineSDNode *Node =
    DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
if (Global) {
  MachinePointerInfo MPInfo(Global);
  auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
               MachineMemOperand::MODereferenceable;
  MachineMemOperand *MemRef = MF.getMachineMemOperand(
      MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy));
  DAG.setNodeMemRefs(Node, {MemRef});
}
if (PtrTy != PtrMemTy)
  return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
return SDValue(Node, 0);
2733}

2735/// Codegen a new tail for a stack protector check ParentMBB which has had its
2736/// tail spliced into a stack protector check success bb.
2737///
2738/// For a high level explanation of how this fits into the stack protector
2739/// generation see the comment on the declaration of class
2740/// StackProtectorDescriptor.
2741void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
                                                MachineBasicBlock *ParentBB) {

// First create the loads to the guard/stack slot for the comparison.
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());

MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
int FI = MFI.getStackProtectorIndex();

SDValue Guard;
SDLoc dl = getCurSDLoc();
SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
const Module &M = *ParentBB->getParent()->getFunction().getParent();
Align Align =
    DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext()));

// Generate code to load the content of the guard slot.
SDValue GuardVal = DAG.getLoad(
    PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
    MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
    MachineMemOperand::MOVolatile);

if (TLI.useStackGuardXorFP())
  GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);

// Retrieve guard check function, nullptr if instrumentation is inlined.
if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
  // The target provides a guard check function to validate the guard value.
  // Generate a call to that function with the content of the guard slot as
  // argument.
  FunctionType *FnTy = GuardCheckFn->getFunctionType();
  assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 &&
 "Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2774
, __extension__ __PRETTY_FUNCTION__));

  TargetLowering::ArgListTy Args;
  TargetLowering::ArgListEntry Entry;
  Entry.Node = GuardVal;
  Entry.Ty = FnTy->getParamType(0);
  if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
    Entry.IsInReg = true;
  Args.push_back(Entry);

  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(getCurSDLoc())
      .setChain(DAG.getEntryNode())
      .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
                 getValue(GuardCheckFn), std::move(Args));

  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
  DAG.setRoot(Result.second);
  return;
}

// If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
// Otherwise, emit a volatile load to retrieve the stack guard value.
SDValue Chain = DAG.getEntryNode();
if (TLI.useLoadStackGuardNode()) {
  Guard = getLoadStackGuard(DAG, dl, Chain);
} else {
  const Value *IRGuard = TLI.getSDagStackGuard(M);
  SDValue GuardPtr = getValue(IRGuard);

  Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
                      MachinePointerInfo(IRGuard, 0), Align,
                      MachineMemOperand::MOVolatile);
}

// Perform the comparison via a getsetcc.
SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
                                                      *DAG.getContext(),
                                                      Guard.getValueType()),
                           Guard, GuardVal, ISD::SETNE);

// If the guard/stackslot do not equal, branch to failure MBB.
SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
                             MVT::Other, GuardVal.getOperand(0),
                             Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
// Otherwise branch to success MBB.
SDValue Br = DAG.getNode(ISD::BR, dl,
                         MVT::Other, BrCond,
                         DAG.getBasicBlock(SPD.getSuccessMBB()));

DAG.setRoot(Br);
2825}

2827/// Codegen the failure basic block for a stack protector check.
2828///
2829/// A failure stack protector machine basic block consists simply of a call to
2830/// __stack_chk_fail().
2831///
2832/// For a high level explanation of how this fits into the stack protector
2833/// generation see the comment on the declaration of class
2834/// StackProtectorDescriptor.
2835void
2836SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
TargetLowering::MakeLibCallOptions CallOptions;
CallOptions.setDiscardResult(true);
SDValue Chain =
    TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
                    std::nullopt, CallOptions, getCurSDLoc())
        .second;
// On PS4/PS5, the "return address" must still be within the calling
// function, even if it's at the very end, so emit an explicit TRAP here.
// Passing 'true' for doesNotReturn above won't generate the trap for us.
if (TM.getTargetTriple().isPS())
  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
// WebAssembly needs an unreachable instruction after a non-returning call,
// because the function return type can be different from __stack_chk_fail's
// return type (void).
if (TM.getTargetTriple().isWasm())
  Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);

DAG.setRoot(Chain);
2856}

2858/// visitBitTestHeader - This function emits necessary code to produce value
2859/// suitable for "bit tests"
2860void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
                                           MachineBasicBlock *SwitchBB) {
SDLoc dl = getCurSDLoc();

// Subtract the minimum value.
SDValue SwitchOp = getValue(B.SValue);
EVT VT = SwitchOp.getValueType();
SDValue RangeSub =
    DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));

// Determine the type of the test operands.
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
bool UsePtrType = false;
if (!TLI.isTypeLegal(VT)) {
  UsePtrType = true;
} else {
  for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
    if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
      // Switch table case range are encoded into series of masks.
      // Just use pointer type, it's guaranteed to fit.
      UsePtrType = true;
      break;
    }
}
SDValue Sub = RangeSub;
if (UsePtrType) {
  VT = TLI.getPointerTy(DAG.getDataLayout());
  Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
}

B.RegVT = VT.getSimpleVT();
B.Reg = FuncInfo.CreateReg(B.RegVT);
SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);

MachineBasicBlock* MBB = B.Cases[0].ThisBB;

if (!B.FallthroughUnreachable)
  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
addSuccessorWithProb(SwitchBB, MBB, B.Prob);
SwitchBB->normalizeSuccProbs();

SDValue Root = CopyTo;
if (!B.FallthroughUnreachable) {
  // Conditional branch to the default block.
  SDValue RangeCmp = DAG.getSetCC(dl,
      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
                             RangeSub.getValueType()),
      RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
      ISD::SETUGT);

  Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
                     DAG.getBasicBlock(B.Default));
}

// Avoid emitting unnecessary branches to the next block.
if (MBB != NextBlock(SwitchBB))
  Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));

DAG.setRoot(Root);
2919}

2921/// visitBitTestCase - this function produces one "bit test"
2922void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
                                         MachineBasicBlock* NextMBB,
                                         BranchProbability BranchProbToNext,
                                         unsigned Reg,
                                         BitTestCase &B,
                                         MachineBasicBlock *SwitchBB) {
SDLoc dl = getCurSDLoc();
MVT VT = BB.RegVT;
SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
SDValue Cmp;
unsigned PopCount = llvm::popcount(B.Mask);
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (PopCount == 1) {
  // Testing for a single bit; just compare the shift count with what it
  // would need to be to shift a 1 bit in that position.
  Cmp = DAG.getSetCC(
      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
      ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
      ISD::SETEQ);
} else if (PopCount == BB.Range) {
  // There is only one zero bit in the range, test for it directly.
  Cmp = DAG.getSetCC(
      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
      ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
} else {
  // Make desired shift
  SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
                                  DAG.getConstant(1, dl, VT), ShiftOp);

  // Emit bit tests and jumps
  SDValue AndOp = DAG.getNode(ISD::AND, dl,
                              VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
  Cmp = DAG.getSetCC(
      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
      AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
}

// The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
// The branch probability from SwitchBB to NextMBB is BranchProbToNext.
addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
// It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
// one as they are relative probabilities (and thus work more like weights),
// and hence we need to normalize them to let the sum of them become one.
SwitchBB->normalizeSuccProbs();

SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
                            MVT::Other, getControlRoot(),
                            Cmp, DAG.getBasicBlock(B.TargetBB));

// Avoid emitting unnecessary branches to the next block.
if (NextMBB != NextBlock(SwitchBB))
  BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
                      DAG.getBasicBlock(NextMBB));

DAG.setRoot(BrAnd);
2978}

2980void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
MachineBasicBlock *InvokeMBB = FuncInfo.MBB;

// Retrieve successors. Look through artificial IR level blocks like
// catchswitch for successors.
MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
const BasicBlock *EHPadBB = I.getSuccessor(1);
MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996
, __extension__ __PRETTY_FUNCTION__))
           {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996
, __extension__ __PRETTY_FUNCTION__))
            LLVMContext::OB_gc_live, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996
, __extension__ __PRETTY_FUNCTION__))
            LLVMContext::OB_cfguardtarget,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996
, __extension__ __PRETTY_FUNCTION__))
            LLVMContext::OB_clang_arc_attachedcall}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996
, __extension__ __PRETTY_FUNCTION__))
       "Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live
, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext
::OB_clang_arc_attachedcall}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, LLVMContext::OB_gc_live, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_clang_arc_attachedcall}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 2996
, __extension__ __PRETTY_FUNCTION__));

const Value *Callee(I.getCalledOperand());
const Function *Fn = dyn_cast<Function>(Callee);
if (isa<InlineAsm>(Callee))
  visitInlineAsm(I, EHPadBB);
else if (Fn && Fn->isIntrinsic()) {
  switch (Fn->getIntrinsicID()) {
  default:
    llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3005
);
  case Intrinsic::donothing:
    // Ignore invokes to @llvm.donothing: jump directly to the next BB.
  case Intrinsic::seh_try_begin:
  case Intrinsic::seh_scope_begin:
  case Intrinsic::seh_try_end:
  case Intrinsic::seh_scope_end:
    if (EHPadMBB)
        // a block referenced by EH table
        // so dtor-funclet not removed by opts
        EHPadMBB->setMachineBlockAddressTaken();
    break;
  case Intrinsic::experimental_patchpoint_void:
  case Intrinsic::experimental_patchpoint_i64:
    visitPatchpoint(I, EHPadBB);
    break;
  case Intrinsic::experimental_gc_statepoint:
    LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
    break;
  case Intrinsic::wasm_rethrow: {
    // This is usually done in visitTargetIntrinsic, but this intrinsic is
    // special because it can be invoked, so we manually lower it to a DAG
    // node here.
    SmallVector<SDValue, 8> Ops;
    Ops.push_back(getRoot()); // inchain
    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    Ops.push_back(
        DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
                              TLI.getPointerTy(DAG.getDataLayout())));
    SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
    DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
    break;
  }
  }
} else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
  // Currently we do not lower any intrinsic calls with deopt operand bundles.
  // Eventually we will support lowering the @llvm.experimental.deoptimize
  // intrinsic, and right now there are no plans to support other intrinsics
  // with deopt state.
  LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
} else {
  LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
}

// If the value of the invoke is used outside of its defining block, make it
// available as a virtual register.
// We already took care of the exported value for the statepoint instruction
// during call to the LowerStatepoint.
if (!isa<GCStatepointInst>(I)) {
  CopyToExportRegsIfNeeded(&I);
}

SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
BranchProbabilityInfo *BPI = FuncInfo.BPI;
BranchProbability EHPadBBProb =
    BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
        : BranchProbability::getZero();
findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);

// Update successor info.
addSuccessorWithProb(InvokeMBB, Return);
for (auto &UnwindDest : UnwindDests) {
  UnwindDest.first->setIsEHPad();
  addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
}
InvokeMBB->normalizeSuccProbs();

// Drop into normal successor.
DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
                        DAG.getBasicBlock(Return)));
3075}

3077void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
MachineBasicBlock *CallBrMBB = FuncInfo.MBB;

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3084
, __extension__ __PRETTY_FUNCTION__))
           {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3084
, __extension__ __PRETTY_FUNCTION__))
       "Cannot lower callbrs with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower callbrs with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower callbrs with arbitrary operand bundles yet!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3084
, __extension__ __PRETTY_FUNCTION__));

assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr")(static_cast <bool> (I.isInlineAsm() && "Only know how to handle inlineasm callbr"
) ? void (0) : __assert_fail ("I.isInlineAsm() && \"Only know how to handle inlineasm callbr\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3086
, __extension__ __PRETTY_FUNCTION__));
visitInlineAsm(I);
CopyToExportRegsIfNeeded(&I);

// Retrieve successors.
SmallPtrSet<BasicBlock *, 8> Dests;
Dests.insert(I.getDefaultDest());
MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];

// Update successor info.
addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
  BasicBlock *Dest = I.getIndirectDest(i);
  MachineBasicBlock *Target = FuncInfo.MBBMap[Dest];
  Target->setIsInlineAsmBrIndirectTarget();
  Target->setMachineBlockAddressTaken();
  Target->setLabelMustBeEmitted();
  // Don't add duplicate machine successors.
  if (Dests.insert(Dest).second)
    addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
}
CallBrMBB->normalizeSuccProbs();

// Drop into default successor.
DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
                        MVT::Other, getControlRoot(),
                        DAG.getBasicBlock(Return)));
3113}

3115void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3116
);
3117}

3119void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
 "Call to landingpad not in landing pad!") ? void (0) : __assert_fail
 ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3121
, __extension__ __PRETTY_FUNCTION__))
       "Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
 "Call to landingpad not in landing pad!") ? void (0) : __assert_fail
 ("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3121
, __extension__ __PRETTY_FUNCTION__));

// If there aren't registers to copy the values into (e.g., during SjLj
// exceptions), then don't bother to create these DAG nodes.
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
    TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
  return;

// If landingpad's return type is token type, we don't create DAG nodes
// for its exception pointer and selector value. The extraction of exception
// pointer or selector value from token type landingpads is not currently
// supported.
if (LP.getType()->isTokenTy())
  return;

SmallVector<EVT, 2> ValueVTs;
SDLoc dl = getCurSDLoc();
ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3141
, __extension__ __PRETTY_FUNCTION__));

// Get the two live-in registers as SDValues. The physregs have already been
// copied into virtual registers.
SDValue Ops[2];
if (FuncInfo.ExceptionPointerVirtReg) {
  Ops[0] = DAG.getZExtOrTrunc(
      DAG.getCopyFromReg(DAG.getEntryNode(), dl,
                         FuncInfo.ExceptionPointerVirtReg,
                         TLI.getPointerTy(DAG.getDataLayout())),
      dl, ValueVTs[0]);
} else {
  Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
}
Ops[1] = DAG.getZExtOrTrunc(
    DAG.getCopyFromReg(DAG.getEntryNode(), dl,
                       FuncInfo.ExceptionSelectorVirtReg,
                       TLI.getPointerTy(DAG.getDataLayout())),
    dl, ValueVTs[1]);

// Merge into one.
SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
                          DAG.getVTList(ValueVTs), Ops);
setValue(&LP, Res);
3165}

3167void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
                                         MachineBasicBlock *Last) {
// Update JTCases.
for (JumpTableBlock &JTB : SL->JTCases)
  if (JTB.first.HeaderBB == First)
    JTB.first.HeaderBB = Last;

// Update BitTestCases.
for (BitTestBlock &BTB : SL->BitTestCases)
  if (BTB.Parent == First)
    BTB.Parent = Last;
3178}

3180void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;

// Update machine-CFG edges with unique successors.
SmallSet<BasicBlock*, 32> Done;
for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
  BasicBlock *BB = I.getSuccessor(i);
  bool Inserted = Done.insert(BB).second;
  if (!Inserted)
      continue;

  MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
  addSuccessorWithProb(IndirectBrMBB, Succ);
}
IndirectBrMBB->normalizeSuccProbs();

DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
                        MVT::Other, getControlRoot(),
                        getValue(I.getAddress())));
3199}

3201void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
if (!DAG.getTarget().Options.TrapUnreachable)
  return;

// We may be able to ignore unreachable behind a noreturn call.
if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
  const BasicBlock &BB = *I.getParent();
  if (&I != &BB.front()) {
    BasicBlock::const_iterator PredI =
      std::prev(BasicBlock::const_iterator(&I));
    if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
      if (Call->doesNotReturn())
        return;
    }
  }
}

DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3219}

3221void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
SDNodeFlags Flags;
if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
  Flags.copyFMF(*FPOp);

SDValue Op = getValue(I.getOperand(0));
SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
                                  Op, Flags);
setValue(&I, UnNodeValue);
3230}

3232void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
SDNodeFlags Flags;
if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
  Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
  Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
}
if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
  Flags.setExact(ExactOp->isExact());
if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
  Flags.copyFMF(*FPOp);

SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));
SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
                                   Op1, Op2, Flags);
setValue(&I, BinNodeValue);
3248}

3250void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));

EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
    Op1.getValueType(), DAG.getDataLayout());

// Coerce the shift amount to the right type if we can. This exposes the
// truncate or zext to optimization early.
if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
  assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) &&(static_cast <bool> (ShiftTy.getSizeInBits() >= Log2_32_Ceil
(Op1.getValueSizeInBits()) && "Unexpected shift type"
) ? void (0) : __assert_fail ("ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && \"Unexpected shift type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3261
, __extension__ __PRETTY_FUNCTION__))
         "Unexpected shift type")(static_cast <bool> (ShiftTy.getSizeInBits() >= Log2_32_Ceil
(Op1.getValueSizeInBits()) && "Unexpected shift type"
) ? void (0) : __assert_fail ("ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && \"Unexpected shift type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3261
, __extension__ __PRETTY_FUNCTION__));
  Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
}

bool nuw = false;
bool nsw = false;
bool exact = false;

if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {

  if (const OverflowingBinaryOperator *OFBinOp =
          dyn_cast<const OverflowingBinaryOperator>(&I)) {
    nuw = OFBinOp->hasNoUnsignedWrap();
    nsw = OFBinOp->hasNoSignedWrap();
  }
  if (const PossiblyExactOperator *ExactOp =
          dyn_cast<const PossiblyExactOperator>(&I))
    exact = ExactOp->isExact();
}
SDNodeFlags Flags;
Flags.setExact(exact);
Flags.setNoSignedWrap(nsw);
Flags.setNoUnsignedWrap(nuw);
SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
                          Flags);
setValue(&I, Res);
3287}

3289void SelectionDAGBuilder::visitSDiv(const User &I) {
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));

SDNodeFlags Flags;
Flags.setExact(isa<PossiblyExactOperator>(&I) &&
               cast<PossiblyExactOperator>(&I)->isExact());
setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
                         Op2, Flags));
3298}

3300void SelectionDAGBuilder::visitICmp(const User &I) {
ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
  predicate = IC->getPredicate();
else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
  predicate = ICmpInst::Predicate(IC->getPredicate());
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));
ISD::CondCode Opcode = getICmpCondCode(predicate);

auto &TLI = DAG.getTargetLoweringInfo();
EVT MemVT =
    TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());

// If a pointer's DAG type is larger than its memory type then the DAG values
// are zero-extended. This breaks signed comparisons so truncate back to the
// underlying type before doing the compare.
if (Op1.getValueType() != MemVT) {
  Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
  Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
}

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3325}

3327void SelectionDAGBuilder::visitFCmp(const User &I) {
FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
  predicate = FC->getPredicate();
else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
  predicate = FCmpInst::Predicate(FC->getPredicate());
SDValue Op1 = getValue(I.getOperand(0));
SDValue Op2 = getValue(I.getOperand(1));

ISD::CondCode Condition = getFCmpCondCode(predicate);
auto *FPMO = cast<FPMathOperator>(&I);
if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
  Condition = getFCmpCodeWithoutNaN(Condition);

SDNodeFlags Flags;
Flags.copyFMF(*FPMO);
SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3348}

3350// Check if the condition of the select has one use or two users that are both
3351// selects with the same condition.
3352static bool hasOnlySelectUsers(const Value *Cond) {
return llvm::all_of(Cond->users(), [](const Value *V) {
  return isa<SelectInst>(V);
});
3356}

3358void SelectionDAGBuilder::visitSelect(const User &I) {
SmallVector<EVT, 4> ValueVTs;
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
                ValueVTs);
unsigned NumValues = ValueVTs.size();
if (NumValues == 0) return;

SmallVector<SDValue, 4> Values(NumValues);
SDValue Cond     = getValue(I.getOperand(0));
SDValue LHSVal   = getValue(I.getOperand(1));
SDValue RHSVal   = getValue(I.getOperand(2));
SmallVector<SDValue, 1> BaseOps(1, Cond);
ISD::NodeType OpCode =
    Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;

bool IsUnaryAbs = false;
bool Negate = false;

SDNodeFlags Flags;
if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
  Flags.copyFMF(*FPOp);

// Min/max matching is only viable if all output VTs are the same.
if (all_equal(ValueVTs)) {
  EVT VT = ValueVTs[0];
  LLVMContext &Ctx = *DAG.getContext();
  auto &TLI = DAG.getTargetLoweringInfo();

  // We care about the legality of the operation after it has been type
  // legalized.
  while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
    VT = TLI.getTypeToTransformTo(Ctx, VT);

  // If the vselect is legal, assume we want to leave this as a vector setcc +
  // vselect. Otherwise, if this is going to be scalarized, we want to see if
  // min/max is legal on the scalar type.
  bool UseScalarMinMax = VT.isVector() &&
    !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);

  // ValueTracking's select pattern matching does not account for -0.0,
  // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
  // -0.0 is less than +0.0.
  Value *LHS, *RHS;
  auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
  ISD::NodeType Opc = ISD::DELETED_NODE;
  switch (SPR.Flavor) {
  case SPF_UMAX:    Opc = ISD::UMAX; break;
  case SPF_UMIN:    Opc = ISD::UMIN; break;
  case SPF_SMAX:    Opc = ISD::SMAX; break;
  case SPF_SMIN:    Opc = ISD::SMIN; break;
  case SPF_FMINNUM:
    switch (SPR.NaNBehavior) {
    case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3410
);
    case SPNB_RETURNS_NAN: break;
    case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
    case SPNB_RETURNS_ANY:
      if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) ||
          (UseScalarMinMax &&
           TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType())))
        Opc = ISD::FMINNUM;
      break;
    }
    break;
  case SPF_FMAXNUM:
    switch (SPR.NaNBehavior) {
    case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3423
);
    case SPNB_RETURNS_NAN: break;
    case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
    case SPNB_RETURNS_ANY:
      if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) ||
          (UseScalarMinMax &&
           TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType())))
        Opc = ISD::FMAXNUM;
      break;
    }
    break;
  case SPF_NABS:
    Negate = true;
    [[fallthrough]];
  case SPF_ABS:
    IsUnaryAbs = true;
    Opc = ISD::ABS;
    break;
  default: break;
  }

  if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
      (TLI.isOperationLegalOrCustom(Opc, VT) ||
       (UseScalarMinMax &&
        TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
      // If the underlying comparison instruction is used by any other
      // instruction, the consumed instructions won't be destroyed, so it is
      // not profitable to convert to a min/max.
      hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
    OpCode = Opc;
    LHSVal = getValue(LHS);
    RHSVal = getValue(RHS);
    BaseOps.clear();
  }

  if (IsUnaryAbs) {
    OpCode = Opc;
    LHSVal = getValue(LHS);
    BaseOps.clear();
  }
}

if (IsUnaryAbs) {
  for (unsigned i = 0; i != NumValues; ++i) {
    SDLoc dl = getCurSDLoc();
    EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
    Values[i] =
        DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
    if (Negate)
      Values[i] = DAG.getNegative(Values[i], dl, VT);
  }
} else {
  for (unsigned i = 0; i != NumValues; ++i) {
    SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
    Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
    Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
    Values[i] = DAG.getNode(
        OpCode, getCurSDLoc(),
        LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
  }
}

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                         DAG.getVTList(ValueVTs), Values));
3487}

3489void SelectionDAGBuilder::visitTrunc(const User &I) {
// TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3495}

3497void SelectionDAGBuilder::visitZExt(const User &I) {
// ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
// ZExt also can't be a cast to bool for same reason. So, nothing much to do
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3504}

3506void SelectionDAGBuilder::visitSExt(const User &I) {
// SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
// SExt also can't be a cast to bool for same reason. So, nothing much to do
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3513}

3515void SelectionDAGBuilder::visitFPTrunc(const User &I) {
// FPTrunc is never a no-op cast, no need to check
SDValue N = getValue(I.getOperand(0));
SDLoc dl = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
                         DAG.getTargetConstant(
                             0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3524}

3526void SelectionDAGBuilder::visitFPExt(const User &I) {
// FPExt is never a no-op cast, no need to check
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3532}

3534void SelectionDAGBuilder::visitFPToUI(const User &I) {
// FPToUI is never a no-op cast, no need to check
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3540}

3542void SelectionDAGBuilder::visitFPToSI(const User &I) {
// FPToSI is never a no-op cast, no need to check
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3548}

3550void SelectionDAGBuilder::visitUIToFP(const User &I) {
// UIToFP is never a no-op cast, no need to check
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3556}

3558void SelectionDAGBuilder::visitSIToFP(const User &I) {
// SIToFP is never a no-op cast, no need to check
SDValue N = getValue(I.getOperand(0));
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3564}

3566void SelectionDAGBuilder::visitPtrToInt(const User &I) {
// What to do depends on the size of the integer and the size of the pointer.
// We can either truncate, zero extend, or no-op, accordingly.
SDValue N = getValue(I.getOperand(0));
auto &TLI = DAG.getTargetLoweringInfo();
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());
EVT PtrMemVT =
    TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
setValue(&I, N);
3578}

3580void SelectionDAGBuilder::visitIntToPtr(const User &I) {
// What to do depends on the size of the integer and the size of the pointer.
// We can either truncate, zero extend, or no-op, accordingly.
SDValue N = getValue(I.getOperand(0));
auto &TLI = DAG.getTargetLoweringInfo();
EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
setValue(&I, N);
3590}

3592void SelectionDAGBuilder::visitBitCast(const User &I) {
SDValue N = getValue(I.getOperand(0));
SDLoc dl = getCurSDLoc();
EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      I.getType());

// BitCast assures us that source and destination are the same size so this is
// either a BITCAST or a no-op.
if (DestVT != N.getValueType())
  setValue(&I, DAG.getNode(ISD::BITCAST, dl,
                           DestVT, N)); // convert types.
// Check if the original LLVM IR Operand was a ConstantInt, because getValue()
// might fold any kind of constant expression to an integer constant and that
// is not what we are looking for. Only recognize a bitcast of a genuine
// constant integer as an opaque constant.
else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
  setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
                               /*isOpaque*/true));
else
  setValue(&I, N);            // noop cast.
3612}

3614void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const Value *SV = I.getOperand(0);
SDValue N = getValue(SV);
EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

unsigned SrcAS = SV->getType()->getPointerAddressSpace();
unsigned DestAS = I.getType()->getPointerAddressSpace();

if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
  N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);

setValue(&I, N);
3627}

3629void SelectionDAGBuilder::visitInsertElement(const User &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue InVec = getValue(I.getOperand(0));
SDValue InVal = getValue(I.getOperand(1));
SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
                         TLI.getValueType(DAG.getDataLayout(), I.getType()),
                         InVec, InVal, InIdx));
3638}

3640void SelectionDAGBuilder::visitExtractElement(const User &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue InVec = getValue(I.getOperand(0));
SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
                         TLI.getValueType(DAG.getDataLayout(), I.getType()),
                         InVec, InIdx));
3648}

3650void SelectionDAGBuilder::visitShuffleVector(const User &I) {
SDValue Src1 = getValue(I.getOperand(0));
SDValue Src2 = getValue(I.getOperand(1));
ArrayRef<int> Mask;
if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
  Mask = SVI->getShuffleMask();
else
  Mask = cast<ConstantExpr>(I).getShuffleMask();
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT SrcVT = Src1.getValueType();

if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
    VT.isScalableVector()) {
  // Canonical splat form of first element of first input vector.
  SDValue FirstElt =
      DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1,
                  DAG.getVectorIdxConstant(0, DL));
  setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
  return;
}

// For now, we only handle splats for scalable vectors.
// The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
// for targets that support a SPLAT_VECTOR for non-scalable vector types.
assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle")(static_cast <bool> (!VT.isScalableVector() && "Unsupported scalable vector shuffle"
) ? void (0) : __assert_fail ("!VT.isScalableVector() && \"Unsupported scalable vector shuffle\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 3676
, __extension__ __PRETTY_FUNCTION__));

unsigned SrcNumElts = SrcVT.getVectorNumElements();
unsigned MaskNumElts = Mask.size();

if (SrcNumElts == MaskNumElts) {
  setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
  return;
}

// Normalize the shuffle vector since mask and vector length don't match.
if (SrcNumElts < MaskNumElts) {
  // Mask is longer than the source vectors. We can use concatenate vector to
  // make the mask and vectors lengths match.

  if (MaskNumElts % SrcNumElts == 0) {
    // Mask length is a multiple of the source vector length.
    // Check if the shuffle is some kind of concatenation of the input
    // vectors.
    unsigned NumConcat = MaskNumElts / SrcNumElts;
    bool IsConcat = true;
    SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
    for (unsigned i = 0; i != MaskNumElts; ++i) {
      int Idx = Mask[i];
      if (Idx < 0)
        continue;
      // Ensure the indices in each SrcVT sized piece are sequential and that
      // the same source is used for the whole piece.
      if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
          (ConcatSrcs[i / SrcNumElts] >= 0 &&
           ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
        IsConcat = false;
        break;
      }
      // Remember which source this index came from.
      ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
    }

    // The shuffle is concatenating multiple vectors together. Just emit
    // a CONCAT_VECTORS operation.
    if (IsConcat) {
      SmallVector<SDValue, 8> ConcatOps;
      for (auto Src : ConcatSrcs) {
        if (Src < 0)
          ConcatOps.push_back(DAG.getUNDEF(SrcVT));
        else if (Src == 0)
          ConcatOps.push_back(Src1);
        else
          ConcatOps.push_back(Src2);
      }
      setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
      return;
    }
  }

  unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
  unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
  EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
                                  PaddedMaskNumElts);

  // Pad both vectors with undefs to make them the same length as the mask.
  SDValue UndefVal = DAG.getUNDEF(SrcVT);

  SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
  SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
  MOps1[0] = Src1;
  MOps2[0] = Src2;

  Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
  Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);

  // Readjust mask for new input vector length.
  SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
  for (unsigned i = 0; i != MaskNumElts; ++i) {
    int Idx = Mask[i];
    if (Idx >= (int)SrcNumElts)
      Idx -= SrcNumElts - PaddedMaskNumElts;
    MappedOps[i] = Idx;
  }

  SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);

  // If the concatenated vector was padded, extract a subvector with the
  // correct number of elements.
  if (MaskNumElts != PaddedMaskNumElts)
    Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
                         DAG.getVectorIdxConstant(0, DL));

  setValue(&I, Result);
  return;
}

if (SrcNumElts > MaskNumElts) {
  // Analyze the access pattern of the vector to see if we can extract
  // two subvectors and do the shuffle.
  int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
  bool CanExtract = true;
  for (int Idx : Mask) {
    unsigned Input = 0;
    if (Idx < 0)
      continue;

    if (Idx >= (int)SrcNumElts) {
      Input = 1;
      Idx -= SrcNumElts;
    }

    // If all the indices come from the same MaskNumElts sized portion of
    // the sources we can use extract. Also make sure the extract wouldn't
    // extract past the end of the source.
    int NewStartIdx = alignDown(Idx, MaskNumElts);
    if (NewStartIdx + MaskNumElts > SrcNumElts ||
        (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
      CanExtract = false;
    // Make sure we always update StartIdx as we use it to track if all
    // elements are undef.
    StartIdx[Input] = NewStartIdx;
  }

  if (StartIdx[0] < 0 && StartIdx[1] < 0) {
    setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
    return;
  }
  if (CanExtract) {
    // Extract appropriate subvector and generate a vector shuffle
    for (unsigned Input = 0; Input < 2; ++Input) {
      SDValue &Src = Input == 0 ? Src1 : Src2;
      if (StartIdx[Input] < 0)
        Src = DAG.getUNDEF(VT);
      else {
        Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
                          DAG.getVectorIdxConstant(StartIdx[Input], DL));
      }
    }

    // Calculate new mask.
    SmallVector<int, 8> MappedOps(Mask);
    for (int &Idx : MappedOps) {
      if (Idx >= (int)SrcNumElts)
        Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
      else if (Idx >= 0)
        Idx -= StartIdx[0];
    }

    setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
    return;
  }
}

// We can't use either concat vectors or extract subvectors so fall back to
// replacing the shuffle with extract and build vector.
// to insert and build vector.
EVT EltVT = VT.getVectorElementType();
SmallVector<SDValue,8> Ops;
for (int Idx : Mask) {
  SDValue Res;

  if (Idx < 0) {
    Res = DAG.getUNDEF(EltVT);
  } else {
    SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
    if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;

    Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
                      DAG.getVectorIdxConstant(Idx, DL));
  }

  Ops.push_back(Res);
}

setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3847}

3849void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
ArrayRef<unsigned> Indices = I.getIndices();
const Value *Op0 = I.getOperand(0);
const Value *Op1 = I.getOperand(1);
Type *AggTy = I.getType();
Type *ValTy = Op1->getType();
bool IntoUndef = isa<UndefValue>(Op0);
bool FromUndef = isa<UndefValue>(Op1);

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SmallVector<EVT, 4> AggValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
SmallVector<EVT, 4> ValValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

unsigned NumAggValues = AggValueVTs.size();
unsigned NumValValues = ValValueVTs.size();
SmallVector<SDValue, 4> Values(NumAggValues);

// Ignore an insertvalue that produces an empty object
if (!NumAggValues) {
  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
  return;
}

SDValue Agg = getValue(Op0);
unsigned i = 0;
// Copy the beginning value(s) from the original aggregate.
for (; i != LinearIndex; ++i)
  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
              SDValue(Agg.getNode(), Agg.getResNo() + i);
// Copy values from the inserted value(s).
if (NumValValues) {
  SDValue Val = getValue(Op1);
  for (; i != LinearIndex + NumValValues; ++i)
    Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
                SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
}
// Copy remaining value(s) from the original aggregate.
for (; i != NumAggValues; ++i)
  Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
              SDValue(Agg.getNode(), Agg.getResNo() + i);

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                         DAG.getVTList(AggValueVTs), Values));
3896}

3898void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
ArrayRef<unsigned> Indices = I.getIndices();
const Value *Op0 = I.getOperand(0);
Type *AggTy = Op0->getType();
Type *ValTy = I.getType();
bool OutOfUndef = isa<UndefValue>(Op0);

unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SmallVector<EVT, 4> ValValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);

unsigned NumValValues = ValValueVTs.size();

// Ignore a extractvalue that produces an empty object
if (!NumValValues) {
  setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
  return;
}

SmallVector<SDValue, 4> Values(NumValValues);

SDValue Agg = getValue(Op0);
// Copy out the selected value(s).
for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
  Values[i - LinearIndex] =
    OutOfUndef ?
      DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
      SDValue(Agg.getNode(), Agg.getResNo() + i);

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                         DAG.getVTList(ValValueVTs), Values));
3931}

3933void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
Value *Op0 = I.getOperand(0);
// Note that the pointer operand may be a vector of pointers. Take the scalar
// element which holds a pointer.
unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
SDValue N = getValue(Op0);
SDLoc dl = getCurSDLoc();
auto &TLI = DAG.getTargetLoweringInfo();

// Normalize Vector GEP - all scalar operands should be converted to the
// splat vector.
bool IsVectorGEP = I.getType()->isVectorTy();
ElementCount VectorElementCount =
    IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
                : ElementCount::getFixed(0);

if (IsVectorGEP && !N.getValueType().isVector()) {
  LLVMContext &Context = *DAG.getContext();
  EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
  N = DAG.getSplat(VT, dl, N);
}

for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
     GTI != E; ++GTI) {
  const Value *Idx = GTI.getOperand();
  if (StructType *StTy = GTI.getStructTypeOrNull()) {
    unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
    if (Field) {
      // N = N + Offset
      uint64_t Offset =
          DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field);

      // In an inbounds GEP with an offset that is nonnegative even when
      // interpreted as signed, assume there is no unsigned overflow.
      SDNodeFlags Flags;
      if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
        Flags.setNoUnsignedWrap(true);

      N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
                      DAG.getConstant(Offset, dl, N.getValueType()), Flags);
    }
  } else {
    // IdxSize is the width of the arithmetic according to IR semantics.
    // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
    // (and fix up the result later).
    unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
    MVT IdxTy = MVT::getIntegerVT(IdxSize);
    TypeSize ElementSize =
        DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType());
    // We intentionally mask away the high bits here; ElementSize may not
    // fit in IdxTy.
    APInt ElementMul(IdxSize, ElementSize.getKnownMinValue());
    bool ElementScalable = ElementSize.isScalable();

    // If this is a scalar constant or a splat vector of constants,
    // handle it quickly.
    const auto *C = dyn_cast<Constant>(Idx);
    if (C && isa<VectorType>(C->getType()))
      C = C->getSplatValue();

    const auto *CI = dyn_cast_or_null<ConstantInt>(C);
    if (CI && CI->isZero())
      continue;
    if (CI && !ElementScalable) {
      APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
      LLVMContext &Context = *DAG.getContext();
      SDValue OffsVal;
      if (IsVectorGEP)
        OffsVal = DAG.getConstant(
            Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
      else
        OffsVal = DAG.getConstant(Offs, dl, IdxTy);

      // In an inbounds GEP with an offset that is nonnegative even when
      // interpreted as signed, assume there is no unsigned overflow.
      SDNodeFlags Flags;
      if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
        Flags.setNoUnsignedWrap(true);

      OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());

      N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
      continue;
    }

    // N = N + Idx * ElementMul;
    SDValue IdxN = getValue(Idx);

    if (!IdxN.getValueType().isVector() && IsVectorGEP) {
      EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(),
                                VectorElementCount);
      IdxN = DAG.getSplat(VT, dl, IdxN);
    }

    // If the index is smaller or larger than intptr_t, truncate or extend
    // it.
    IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());

    if (ElementScalable) {
      EVT VScaleTy = N.getValueType().getScalarType();
      SDValue VScale = DAG.getNode(
          ISD::VSCALE, dl, VScaleTy,
          DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
      if (IsVectorGEP)
        VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
      IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale);
    } else {
      // If this is a multiply by a power of two, turn it into a shl
      // immediately.  This is a very common case.
      if (ElementMul != 1) {
        if (ElementMul.isPowerOf2()) {
          unsigned Amt = ElementMul.logBase2();
          IdxN = DAG.getNode(ISD::SHL, dl,
                             N.getValueType(), IdxN,
                             DAG.getConstant(Amt, dl, IdxN.getValueType()));
        } else {
          SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
                                          IdxN.getValueType());
          IdxN = DAG.getNode(ISD::MUL, dl,
                             N.getValueType(), IdxN, Scale);
        }
      }
    }

    N = DAG.getNode(ISD::ADD, dl,
                    N.getValueType(), N, IdxN);
  }
}

MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
if (IsVectorGEP) {
  PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
  PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
}

if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
  N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);

setValue(&I, N);
4073}

4075void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
// If this is a fixed sized alloca in the entry block of the function,
// allocate it statically on the stack.
if (FuncInfo.StaticAllocaMap.count(&I))
  return;   // getValue will auto-populate this.

SDLoc dl = getCurSDLoc();
Type *Ty = I.getAllocatedType();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto &DL = DAG.getDataLayout();
TypeSize TySize = DL.getTypeAllocSize(Ty);
MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());

SDValue AllocSize = getValue(I.getArraySize());

EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace());
if (AllocSize.getValueType() != IntPtr)
  AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);

if (TySize.isScalable())
  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
                          DAG.getVScale(dl, IntPtr,
                                        APInt(IntPtr.getScalarSizeInBits(),
                                              TySize.getKnownMinValue())));
else
  AllocSize =
      DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
                  DAG.getConstant(TySize.getFixedValue(), dl, IntPtr));

// Handle alignment.  If the requested alignment is less than or equal to
// the stack alignment, ignore it.  If the size is greater than or equal to
// the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign();
if (*Alignment <= StackAlign)
  Alignment = std::nullopt;

const uint64_t StackAlignMask = StackAlign.value() - 1U;
// Round the size of the allocation up to the stack alignment size
// by add SA-1 to the size. This doesn't overflow because we're computing
// an address inside an alloca.
SDNodeFlags Flags;
Flags.setNoUnsignedWrap(true);
AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
                        DAG.getConstant(StackAlignMask, dl, IntPtr), Flags);

// Mask out the low bits for alignment purposes.
AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
                        DAG.getConstant(~StackAlignMask, dl, IntPtr));

SDValue Ops[] = {
    getRoot(), AllocSize,
    DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
setValue(&I, DSA);
DAG.setRoot(DSA.getValue(1));

assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects
()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4132
, __extension__ __PRETTY_FUNCTION__));
4133}

4135void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
if (I.isAtomic())
  return visitAtomicLoad(I);

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const Value *SV = I.getOperand(0);
if (TLI.supportSwiftError()) {
  // Swifterror values can come from either a function parameter with
  // swifterror attribute or an alloca with swifterror attribute.
  if (const Argument *Arg = dyn_cast<Argument>(SV)) {
    if (Arg->hasSwiftErrorAttr())
      return visitLoadFromSwiftError(I);
  }

  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
    if (Alloca->isSwiftError())
      return visitLoadFromSwiftError(I);
  }
}

SDValue Ptr = getValue(SV);

Type *Ty = I.getType();
SmallVector<EVT, 4> ValueVTs, MemVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
unsigned NumValues = ValueVTs.size();
if (NumValues == 0)
  return;

Align Alignment = I.getAlign();
AAMDNodes AAInfo = I.getAAMetadata();
const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
bool isVolatile = I.isVolatile();
MachineMemOperand::Flags MMOFlags =
    TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);

SDValue Root;
bool ConstantMemory = false;
if (isVolatile)
  // Serialize volatile loads with other side effects.
  Root = getRoot();
else if (NumValues > MaxParallelChains)
  Root = getMemoryRoot();
else if (AA &&
         AA->pointsToConstantMemory(MemoryLocation(
             SV,
             LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
             AAInfo))) {
  // Do not serialize (non-volatile) loads of constant memory with anything.
  Root = DAG.getEntryNode();
  ConstantMemory = true;
  MMOFlags |= MachineMemOperand::MOInvariant;
} else {
  // Do not serialize non-volatile loads against each other.
  Root = DAG.getRoot();
}

SDLoc dl = getCurSDLoc();

if (isVolatile)
  Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);

// An aggregate load cannot wrap around the address space, so offsets to its
// parts don't wrap either.
SDNodeFlags Flags;
Flags.setNoUnsignedWrap(true);

SmallVector<SDValue, 4> Values(NumValues);
SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
EVT PtrVT = Ptr.getValueType();

unsigned ChainI = 0;
for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
  // Serializing loads here may result in excessive register pressure, and
  // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
  // could recover a bit by hoisting nodes upward in the chain by recognizing
  // they are side-effect free or do not alias. The optimizer should really
  // avoid this case by converting large object/array copies to llvm.memcpy
  // (MaxParallelChains should always remain as failsafe).
  if (ChainI == MaxParallelChains) {
    assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first"
) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4216
, __extension__ __PRETTY_FUNCTION__));
    SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                ArrayRef(Chains.data(), ChainI));
    Root = Chain;
    ChainI = 0;
  }
  SDValue A = DAG.getNode(ISD::ADD, dl,
                          PtrVT, Ptr,
                          DAG.getConstant(Offsets[i], dl, PtrVT),
                          Flags);

  SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
                          MachinePointerInfo(SV, Offsets[i]), Alignment,
                          MMOFlags, AAInfo, Ranges);
  Chains[ChainI] = L.getValue(1);

  if (MemVTs[i] != ValueVTs[i])
    L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);

  Values[i] = L;
}

if (!ConstantMemory) {
  SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                              ArrayRef(Chains.data(), ChainI));
  if (isVolatile)
    DAG.setRoot(Chain);
  else
    PendingLoads.push_back(Chain);
}

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
                         DAG.getVTList(ValueVTs), Values));
4249}

4251void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4253
, __extension__ __PRETTY_FUNCTION__))
       "call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4253
, __extension__ __PRETTY_FUNCTION__));

SmallVector<EVT, 4> ValueVTs;
SmallVector<uint64_t, 4> Offsets;
const Value *SrcV = I.getOperand(0);
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
                SrcV->getType(), ValueVTs, &Offsets);
assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
 (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4261
, __extension__ __PRETTY_FUNCTION__))
       "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
 (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4261
, __extension__ __PRETTY_FUNCTION__));

SDValue Src = getValue(SrcV);
// Create a virtual register, then update the virtual register.
Register VReg =
    SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
// Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
// Chain can be getRoot or getControlRoot.
SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
                                    SDValue(Src.getNode(), Src.getResNo()));
DAG.setRoot(CopyNode);
4272}

4274void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4276
, __extension__ __PRETTY_FUNCTION__))
       "call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4276
, __extension__ __PRETTY_FUNCTION__));

assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281
, __extension__ __PRETTY_FUNCTION__))
       !I.hasMetadata(LLVMContext::MD_nontemporal) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281
, __extension__ __PRETTY_FUNCTION__))
       !I.hasMetadata(LLVMContext::MD_invariant_load) &&(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281
, __extension__ __PRETTY_FUNCTION__))
       "Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && !I.hasMetadata
(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext
::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4281
, __extension__ __PRETTY_FUNCTION__));

const Value *SV = I.getOperand(0);
Type *Ty = I.getType();
assert((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290
, __extension__ __PRETTY_FUNCTION__))
    (!AA ||(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290
, __extension__ __PRETTY_FUNCTION__))
     !AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290
, __extension__ __PRETTY_FUNCTION__))
         SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290
, __extension__ __PRETTY_FUNCTION__))
         I.getAAMetadata()))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290
, __extension__ __PRETTY_FUNCTION__))
    "load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout(
).getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && \"load_from_swift_error should not be constant memory\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4290
, __extension__ __PRETTY_FUNCTION__));

SmallVector<EVT, 4> ValueVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
                ValueVTs, &Offsets);
assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
 (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4297
, __extension__ __PRETTY_FUNCTION__))
       "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
 (0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4297
, __extension__ __PRETTY_FUNCTION__));

// Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
SDValue L = DAG.getCopyFromReg(
    getRoot(), getCurSDLoc(),
    SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);

setValue(&I, L);
4305}

4307void SelectionDAGBuilder::visitStore(const StoreInst &I) {
if (I.isAtomic())
  return visitAtomicStore(I);

const Value *SrcV = I.getOperand(0);
const Value *PtrV = I.getOperand(1);

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (TLI.supportSwiftError()) {
  // Swifterror values can come from either a function parameter with
  // swifterror attribute or an alloca with swifterror attribute.
  if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
    if (Arg->hasSwiftErrorAttr())
      return visitStoreToSwiftError(I);
  }

  if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
    if (Alloca->isSwiftError())
      return visitStoreToSwiftError(I);
  }
}

SmallVector<EVT, 4> ValueVTs, MemVTs;
SmallVector<uint64_t, 4> Offsets;
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
                SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
unsigned NumValues = ValueVTs.size();
if (NumValues == 0)
  return;

// Get the lowered operands. Note that we do this after
// checking if NumResults is zero, because with zero results
// the operands won't have values in the map.
SDValue Src = getValue(SrcV);
SDValue Ptr = getValue(PtrV);

SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
SDLoc dl = getCurSDLoc();
Align Alignment = I.getAlign();
AAMDNodes AAInfo = I.getAAMetadata();

auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

// An aggregate load cannot wrap around the address space, so offsets to its
// parts don't wrap either.
SDNodeFlags Flags;
Flags.setNoUnsignedWrap(true);

unsigned ChainI = 0;
for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
  // See visitLoad comments.
  if (ChainI == MaxParallelChains) {
    SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                ArrayRef(Chains.data(), ChainI));
    Root = Chain;
    ChainI = 0;
  }
  SDValue Add =
      DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags);
  SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
  if (MemVTs[i] != ValueVTs[i])
    Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
  SDValue St =
      DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
                   Alignment, MMOFlags, AAInfo);
  Chains[ChainI] = St;
}

SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                                ArrayRef(Chains.data(), ChainI));
setValue(&I, StoreNode);
DAG.setRoot(StoreNode);
4380}

4382void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
                                         bool IsCompressing) {
SDLoc sdl = getCurSDLoc();

auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
                             MaybeAlign &Alignment) {
  // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
  Src0 = I.getArgOperand(0);
  Ptr = I.getArgOperand(1);
  Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue();
  Mask = I.getArgOperand(3);
};
auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
                                  MaybeAlign &Alignment) {
  // llvm.masked.compressstore.*(Src0, Ptr, Mask)
  Src0 = I.getArgOperand(0);
  Ptr = I.getArgOperand(1);
  Mask = I.getArgOperand(2);
  Alignment = std::nullopt;
};

Value  *PtrOperand, *MaskOperand, *Src0Operand;
MaybeAlign Alignment;
if (IsCompressing)
  getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
else
  getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

SDValue Ptr = getValue(PtrOperand);
SDValue Src0 = getValue(Src0Operand);
SDValue Mask = getValue(MaskOperand);
SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

EVT VT = Src0.getValueType();
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT);

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
    MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata());
SDValue StoreNode =
    DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO,
                       ISD::UNINDEXED, false /* Truncating */, IsCompressing);
DAG.setRoot(StoreNode);
setValue(&I, StoreNode);
4427}

4429// Get a uniform base for the Gather/Scatter intrinsic.
4430// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4431// We try to represent it as a base pointer + vector of indices.
4432// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4433// The first operand of the GEP may be a single pointer or a vector of pointers
4434// Example:
4435//   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4436//  or
4437//   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
4438// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4439//
4440// When the first GEP operand is a single pointer - it is the uniform base we
4441// are looking for. If first operand of the GEP is a splat vector - we
4442// extract the splat value and use it as a uniform base.
4443// In all other cases the function returns 'false'.
4444static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
                         ISD::MemIndexType &IndexType, SDValue &Scale,
                         SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
                         uint64_t ElemSize) {
SelectionDAG& DAG = SDB->DAG;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const DataLayout &DL = DAG.getDataLayout();

assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
 && "Unexpected pointer type") ? void (0) : __assert_fail
 ("Ptr->getType()->isVectorTy() && \"Unexpected pointer type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4452
, __extension__ __PRETTY_FUNCTION__));

// Handle splat constant pointer.
if (auto *C = dyn_cast<Constant>(Ptr)) {
  C = C->getSplatValue();
  if (!C)
    return false;

  Base = SDB->getValue(C);

  ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
  EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
  Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
  IndexType = ISD::SIGNED_SCALED;
  Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
  return true;
}

const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
if (!GEP || GEP->getParent() != CurBB)
  return false;

if (GEP->getNumOperands() != 2)
  return false;

const Value *BasePtr = GEP->getPointerOperand();
const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);

// Make sure the base is scalar and the index is a vector.
if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
  return false;

uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());

// Target may not support the required addressing mode.
if (ScaleVal != 1 &&
    !TLI.isLegalScaleForGatherScatter(ScaleVal, ElemSize))
  return false;

Base = SDB->getValue(BasePtr);
Index = SDB->getValue(IndexVal);
IndexType = ISD::SIGNED_SCALED;

Scale =
    DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
return true;
4498}

4500void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
SDLoc sdl = getCurSDLoc();

// llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
const Value *Ptr = I.getArgOperand(1);
SDValue Src0 = getValue(I.getArgOperand(0));
SDValue Mask = getValue(I.getArgOperand(3));
EVT VT = Src0.getValueType();
Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
                      ->getMaybeAlignValue()
                      .value_or(DAG.getEVTAlign(VT.getScalarType()));
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

SDValue Base;
SDValue Index;
ISD::MemIndexType IndexType;
SDValue Scale;
bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
                                  I.getParent(), VT.getScalarStoreSize());

unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(AS), MachineMemOperand::MOStore,
    // TODO: Make MachineMemOperands aware of scalable
    // vectors.
    MemoryLocation::UnknownSize, Alignment, I.getAAMetadata());
if (!UniformBase) {
  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
  Index = getValue(Ptr);
  IndexType = ISD::SIGNED_SCALED;
  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
}

EVT IdxVT = Index.getValueType();
EVT EltTy = IdxVT.getVectorElementType();
if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
  EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
  Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
}

SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
                                       Ops, MMO, IndexType, false);
DAG.setRoot(Scatter);
setValue(&I, Scatter);
4545}

4547void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
SDLoc sdl = getCurSDLoc();

auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
                            MaybeAlign &Alignment) {
  // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
  Ptr = I.getArgOperand(0);
  Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue();
  Mask = I.getArgOperand(2);
  Src0 = I.getArgOperand(3);
};
auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
                               MaybeAlign &Alignment) {
  // @llvm.masked.expandload.*(Ptr, Mask, Src0)
  Ptr = I.getArgOperand(0);
  Alignment = std::nullopt;
  Mask = I.getArgOperand(1);
  Src0 = I.getArgOperand(2);
};

Value  *PtrOperand, *MaskOperand, *Src0Operand;
MaybeAlign Alignment;
if (IsExpanding)
  getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
else
  getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);

SDValue Ptr = getValue(PtrOperand);
SDValue Src0 = getValue(Src0Operand);
SDValue Mask = getValue(MaskOperand);
SDValue Offset = DAG.getUNDEF(Ptr.getValueType());

EVT VT = Src0.getValueType();
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT);

AAMDNodes AAInfo = I.getAAMetadata();
const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

// Do not serialize masked loads of constant memory with anything.
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);

SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
    MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);

SDValue Load =
    DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
                      ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
if (AddToChain)
  PendingLoads.push_back(Load.getValue(1));
setValue(&I, Load);
4602}

4604void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
SDLoc sdl = getCurSDLoc();

// @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
const Value *Ptr = I.getArgOperand(0);
SDValue Src0 = getValue(I.getArgOperand(3));
SDValue Mask = getValue(I.getArgOperand(2));

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
                      ->getMaybeAlignValue()
                      .value_or(DAG.getEVTAlign(VT.getScalarType()));

const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);

SDValue Root = DAG.getRoot();
SDValue Base;
SDValue Index;
ISD::MemIndexType IndexType;
SDValue Scale;
bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
                                  I.getParent(), VT.getScalarStoreSize());
unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(AS), MachineMemOperand::MOLoad,
    // TODO: Make MachineMemOperands aware of scalable
    // vectors.
    MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);

if (!UniformBase) {
  Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
  Index = getValue(Ptr);
  IndexType = ISD::SIGNED_SCALED;
  Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
}

EVT IdxVT = Index.getValueType();
EVT EltTy = IdxVT.getVectorElementType();
if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
  EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
  Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
}

SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
                                     Ops, MMO, IndexType, ISD::NON_EXTLOAD);

PendingLoads.push_back(Gather.getValue(1));
setValue(&I, Gather);
4654}

4656void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
SDLoc dl = getCurSDLoc();
AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
AtomicOrdering FailureOrdering = I.getFailureOrdering();
SyncScope::ID SSID = I.getSyncScopeID();

SDValue InChain = getRoot();

MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
    MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
    DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering,
    FailureOrdering);

SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
                                 dl, MemVT, VTs, InChain,
                                 getValue(I.getPointerOperand()),
                                 getValue(I.getCompareOperand()),
                                 getValue(I.getNewValOperand()), MMO);

SDValue OutChain = L.getValue(2);

setValue(&I, L);
DAG.setRoot(OutChain);
4686}

4688void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
SDLoc dl = getCurSDLoc();
ISD::NodeType NT;
switch (I.getOperation()) {
default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4692
);
case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
case AtomicRMWInst::Add:  NT = ISD::ATOMIC_LOAD_ADD; break;
case AtomicRMWInst::Sub:  NT = ISD::ATOMIC_LOAD_SUB; break;
case AtomicRMWInst::And:  NT = ISD::ATOMIC_LOAD_AND; break;
case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
case AtomicRMWInst::Or:   NT = ISD::ATOMIC_LOAD_OR; break;
case AtomicRMWInst::Xor:  NT = ISD::ATOMIC_LOAD_XOR; break;
case AtomicRMWInst::Max:  NT = ISD::ATOMIC_LOAD_MAX; break;
case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break;
case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break;
case AtomicRMWInst::UIncWrap:
  NT = ISD::ATOMIC_LOAD_UINC_WRAP;
  break;
case AtomicRMWInst::UDecWrap:
  NT = ISD::ATOMIC_LOAD_UDEC_WRAP;
  break;
}
AtomicOrdering Ordering = I.getOrdering();
SyncScope::ID SSID = I.getSyncScopeID();

SDValue InChain = getRoot();

auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());

MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
    MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
    DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering);

SDValue L =
  DAG.getAtomic(NT, dl, MemVT, InChain,
                getValue(I.getPointerOperand()), getValue(I.getValOperand()),
                MMO);

SDValue OutChain = L.getValue(1);

setValue(&I, L);
DAG.setRoot(OutChain);
4738}

4740void SelectionDAGBuilder::visitFence(const FenceInst &I) {
SDLoc dl = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue Ops[3];
Ops[0] = getRoot();
Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
                               TLI.getFenceOperandTy(DAG.getDataLayout()));
Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
                               TLI.getFenceOperandTy(DAG.getDataLayout()));
SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
setValue(&I, N);
DAG.setRoot(N);
4752}

4754void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
SDLoc dl = getCurSDLoc();
AtomicOrdering Order = I.getOrdering();
SyncScope::ID SSID = I.getSyncScopeID();

SDValue InChain = getRoot();

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());

if (!TLI.supportsUnalignedAtomics() &&
    I.getAlign().value() < MemVT.getSizeInBits() / 8)
  report_fatal_error("Cannot generate unaligned atomic load");

auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo);

MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
    I.getAlign(), AAMDNodes(), nullptr, SSID, Order);

InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

SDValue Ptr = getValue(I.getPointerOperand());

if (TLI.lowerAtomicLoadAsLoadSDNode(I)) {
  // TODO: Once this is better exercised by tests, it should be merged with
  // the normal path for loads to prevent future divergence.
  SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO);
  if (MemVT != VT)
    L = DAG.getPtrExtOrTrunc(L, dl, VT);

  setValue(&I, L);
  SDValue OutChain = L.getValue(1);
  if (!I.isUnordered())
    DAG.setRoot(OutChain);
  else
    PendingLoads.push_back(OutChain);
  return;
}

SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
                          Ptr, MMO);

SDValue OutChain = L.getValue(1);
if (MemVT != VT)
  L = DAG.getPtrExtOrTrunc(L, dl, VT);

setValue(&I, L);
DAG.setRoot(OutChain);
4804}

4806void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
SDLoc dl = getCurSDLoc();

AtomicOrdering Ordering = I.getOrdering();
SyncScope::ID SSID = I.getSyncScopeID();

SDValue InChain = getRoot();

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT MemVT =
    TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());

if (!TLI.supportsUnalignedAtomics() &&
    I.getAlign().value() < MemVT.getSizeInBits() / 8)
  report_fatal_error("Cannot generate unaligned atomic store");

auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());

MachineFunction &MF = DAG.getMachineFunction();
MachineMemOperand *MMO = MF.getMachineMemOperand(
    MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(),
    I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering);

SDValue Val = getValue(I.getValueOperand());
if (Val.getValueType() != MemVT)
  Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
SDValue Ptr = getValue(I.getPointerOperand());

if (TLI.lowerAtomicStoreAsStoreSDNode(I)) {
  // TODO: Once this is better exercised by tests, it should be merged with
  // the normal path for stores to prevent future divergence.
  SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO);
  setValue(&I, S);
  DAG.setRoot(S);
  return;
}
SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
                                 Ptr, Val, MMO);

setValue(&I, OutChain);
DAG.setRoot(OutChain);
4847}

4849/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4850/// node.
4851void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
                                             unsigned Intrinsic) {
// Ignore the callsite's attributes. A specific call site may be marked with
// readnone, but the lowering code will expect the chain based on the
// definition.
const Function *F = I.getCalledFunction();
bool HasChain = !F->doesNotAccessMemory();
bool OnlyLoad = HasChain && F->onlyReadsMemory();

// Build the operand list.
SmallVector<SDValue, 8> Ops;
if (HasChain) {  // If this intrinsic has side-effects, chainify it.
  if (OnlyLoad) {
    // We don't need to serialize loads against other loads.
    Ops.push_back(DAG.getRoot());
  } else {
    Ops.push_back(getRoot());
  }
}

// Info is set by getTgtMemIntrinsic
TargetLowering::IntrinsicInfo Info;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
                                             DAG.getMachineFunction(),
                                             Intrinsic);

// Add the intrinsic ID as an integer operand if it's not a target intrinsic.
if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
    Info.opc == ISD::INTRINSIC_W_CHAIN)
  Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
                                      TLI.getPointerTy(DAG.getDataLayout())));

// Add all operands of the call to the operand list.
for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
  const Value *Arg = I.getArgOperand(i);
  if (!I.paramHasAttr(i, Attribute::ImmArg)) {
    Ops.push_back(getValue(Arg));
    continue;
  }

  // Use TargetConstant instead of a regular constant for immarg.
  EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
  if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
    assert(CI->getBitWidth() <= 64 &&(static_cast <bool> (CI->getBitWidth() <= 64 &&
 "large intrinsic immediates not handled") ? void (0) : __assert_fail
 ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4896
, __extension__ __PRETTY_FUNCTION__))
           "large intrinsic immediates not handled")(static_cast <bool> (CI->getBitWidth() <= 64 &&
 "large intrinsic immediates not handled") ? void (0) : __assert_fail
 ("CI->getBitWidth() <= 64 && \"large intrinsic immediates not handled\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 4896
, __extension__ __PRETTY_FUNCTION__));
    Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
  } else {
    Ops.push_back(
        DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
  }
}

SmallVector<EVT, 4> ValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);

if (HasChain)
  ValueVTs.push_back(MVT::Other);

SDVTList VTs = DAG.getVTList(ValueVTs);

// Propagate fast-math-flags from IR to node(s).
SDNodeFlags Flags;
if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
  Flags.copyFMF(*FPMO);
SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

// Create the node.
SDValue Result;
// In some cases, custom collection of operands from CallInst I may be needed.
TLI.CollectTargetIntrinsicOperands(I, Ops, DAG);
if (IsTgtIntrinsic) {
  // This is target intrinsic that touches memory
  //
  // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
  //       didn't yield anything useful.
  MachinePointerInfo MPI;
  if (Info.ptrVal)
    MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
  else if (Info.fallbackAddressSpace)
    MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
  Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
                                   Info.memVT, MPI, Info.align, Info.flags,
                                   Info.size, I.getAAMetadata());
} else if (!HasChain) {
  Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
} else if (!I.getType()->isVoidTy()) {
  Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
} else {
  Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
}

if (HasChain) {
  SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
  if (OnlyLoad)
    PendingLoads.push_back(Chain);
  else
    DAG.setRoot(Chain);
}

if (!I.getType()->isVoidTy()) {
  if (!isa<VectorType>(I.getType()))
    Result = lowerRangeToAssertZExt(DAG, I, Result);

  MaybeAlign Alignment = I.getRetAlign();

  // Insert `assertalign` node if there's an alignment.
  if (InsertAssertAlign && Alignment) {
    Result =
        DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
  }

  setValue(&I, Result);
}
4965}

4967/// GetSignificand - Get the significand and build it into a floating-point
4968/// number with exponent of 1:
4969///
4970///   Op = (Op & 0x007fffff) | 0x3f800000;
4971///
4972/// where Op is the hexadecimal representation of floating point value.
4973static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
                         DAG.getConstant(0x007fffff, dl, MVT::i32));
SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
                         DAG.getConstant(0x3f800000, dl, MVT::i32));
return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4979}

4981/// GetExponent - Get the exponent:
4982///
4983///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4984///
4985/// where Op is the hexadecimal representation of floating point value.
4986static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
                         const TargetLowering &TLI, const SDLoc &dl) {
SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
                         DAG.getConstant(0x7f800000, dl, MVT::i32));
SDValue t1 = DAG.getNode(
    ISD::SRL, dl, MVT::i32, t0,
    DAG.getConstant(23, dl,
                    TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
                         DAG.getConstant(127, dl, MVT::i32));
return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4997}

4999/// getF32Constant - Get 32-bit floating point constant.
5000static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
                            const SDLoc &dl) {
return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
                         MVT::f32);
5004}

5006static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
                                     SelectionDAG &DAG) {
// TODO: What fast-math-flags should be set on the floating-point nodes?

//   IntegerPartOfX = ((int32_t)(t0);
SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);

//   FractionalPartOfX = t0 - (float)IntegerPartOfX;
SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);

//   IntegerPartOfX <<= 23;
IntegerPartOfX =
    DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
                DAG.getConstant(23, dl,
                                DAG.getTargetLoweringInfo().getShiftAmountTy(
                                    MVT::i32, DAG.getDataLayout())));

SDValue TwoToFractionalPartOfX;
if (LimitFloatPrecision <= 6) {
  // For floating-point precision of 6:
  //
  //   TwoToFractionalPartOfX =
  //     0.997535578f +
  //       (0.735607626f + 0.252464424f * x) * x;
  //
  // error 0.0144103317, which is 6 bits
  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                           getF32Constant(DAG, 0x3e814304, dl));
  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
                           getF32Constant(DAG, 0x3f3c50c8, dl));
  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                                       getF32Constant(DAG, 0x3f7f5e7e, dl));
} else if (LimitFloatPrecision <= 12) {
  // For floating-point precision of 12:
  //
  //   TwoToFractionalPartOfX =
  //     0.999892986f +
  //       (0.696457318f +
  //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
  //
  // error 0.000107046256, which is 13 to 14 bits
  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                           getF32Constant(DAG, 0x3da235e3, dl));
  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
                           getF32Constant(DAG, 0x3e65b8f3, dl));
  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                           getF32Constant(DAG, 0x3f324b07, dl));
  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
                                       getF32Constant(DAG, 0x3f7ff8fd, dl));
} else { // LimitFloatPrecision <= 18
  // For floating-point precision of 18:
  //
  //   TwoToFractionalPartOfX =
  //     0.999999982f +
  //       (0.693148872f +
  //         (0.240227044f +
  //           (0.554906021e-1f +
  //             (0.961591928e-2f +
  //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
  // error 2.47208000*10^(-7), which is better than 18 bits
  SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                           getF32Constant(DAG, 0x3924b03e, dl));
  SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
                           getF32Constant(DAG, 0x3ab24b87, dl));
  SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
  SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                           getF32Constant(DAG, 0x3c1d8c17, dl));
  SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
  SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
                           getF32Constant(DAG, 0x3d634a1d, dl));
  SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
  SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
                           getF32Constant(DAG, 0x3e75fe14, dl));
  SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
  SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
                            getF32Constant(DAG, 0x3f317234, dl));
  SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
  TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
                                       getF32Constant(DAG, 0x3f800000, dl));
}

// Add the exponent into the result in integer domain.
SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
                   DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5095}

5097/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5098/// limited-precision mode.
5099static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
                       const TargetLowering &TLI, SDNodeFlags Flags) {
if (Op.getValueType() == MVT::f32 &&
    LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {

  // Put the exponent in the right bit position for later addition to the
  // final result:
  //
  // t0 = Op * log2(e)

  // TODO: What fast-math-flags should be set here?
  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
                           DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
  return getLimitedPrecisionExp2(t0, dl, DAG);
}

// No special expansion.
return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5117}

5119/// expandLog - Lower a log intrinsic. Handles the special sequences for
5120/// limited-precision mode.
5121static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
                       const TargetLowering &TLI, SDNodeFlags Flags) {
// TODO: What fast-math-flags should be set on the floating-point nodes?

if (Op.getValueType() == MVT::f32 &&
    LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

  // Scale the exponent by log(2).
  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
  SDValue LogOfExponent =
      DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
                  DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));

  // Get the significand and build it into a floating-point number with
  // exponent of 1.
  SDValue X = GetSignificand(DAG, Op1, dl);

  SDValue LogOfMantissa;
  if (LimitFloatPrecision <= 6) {
    // For floating-point precision of 6:
    //
    //   LogofMantissa =
    //     -1.1609546f +
    //       (1.4034025f - 0.23903021f * x) * x;
    //
    // error 0.0034276066, which is better than 8 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbe74c456, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3fb3a2b1, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                                getF32Constant(DAG, 0x3f949a29, dl));
  } else if (LimitFloatPrecision <= 12) {
    // For floating-point precision of 12:
    //
    //   LogOfMantissa =
    //     -1.7417939f +
    //       (2.8212026f +
    //         (-1.4699568f +
    //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
    //
    // error 0.000061011436, which is 14 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbd67b6d6, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3ee4f4b8, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                             getF32Constant(DAG, 0x3fbc278b, dl));
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
    SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                             getF32Constant(DAG, 0x40348e95, dl));
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
    LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
                                getF32Constant(DAG, 0x3fdef31a, dl));
  } else { // LimitFloatPrecision <= 18
    // For floating-point precision of 18:
    //
    //   LogOfMantissa =
    //     -2.1072184f +
    //       (4.2372794f +
    //         (-3.7029485f +
    //           (2.2781945f +
    //             (-0.87823314f +
    //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
    //
    // error 0.0000023660568, which is better than 18 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbc91e5ac, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3e4350aa, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                             getF32Constant(DAG, 0x3f60d3e3, dl));
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
    SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                             getF32Constant(DAG, 0x4011cdf0, dl));
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
    SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
                             getF32Constant(DAG, 0x406cfd1c, dl));
    SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
    SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
                             getF32Constant(DAG, 0x408797cb, dl));
    SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
    LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
                                getF32Constant(DAG, 0x4006dcab, dl));
  }

  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
}

// No special expansion.
return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5216}

5218/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5219/// limited-precision mode.
5220static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
                        const TargetLowering &TLI, SDNodeFlags Flags) {
// TODO: What fast-math-flags should be set on the floating-point nodes?

if (Op.getValueType() == MVT::f32 &&
    LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

  // Get the exponent.
  SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);

  // Get the significand and build it into a floating-point number with
  // exponent of 1.
  SDValue X = GetSignificand(DAG, Op1, dl);

  // Different possible minimax approximations of significand in
  // floating-point for various degrees of accuracy over [1,2].
  SDValue Log2ofMantissa;
  if (LimitFloatPrecision <= 6) {
    // For floating-point precision of 6:
    //
    //   Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
    //
    // error 0.0049451742, which is more than 7 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbeb08fe0, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x40019463, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                                 getF32Constant(DAG, 0x3fd6633d, dl));
  } else if (LimitFloatPrecision <= 12) {
    // For floating-point precision of 12:
    //
    //   Log2ofMantissa =
    //     -2.51285454f +
    //       (4.07009056f +
    //         (-2.12067489f +
    //           (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
    //
    // error 0.0000876136000, which is better than 13 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbda7262e, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3f25280b, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                             getF32Constant(DAG, 0x4007b923, dl));
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
    SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                             getF32Constant(DAG, 0x40823e2f, dl));
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
    Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
                                 getF32Constant(DAG, 0x4020d29c, dl));
  } else { // LimitFloatPrecision <= 18
    // For floating-point precision of 18:
    //
    //   Log2ofMantissa =
    //     -3.0400495f +
    //       (6.1129976f +
    //         (-5.3420409f +
    //           (3.2865683f +
    //             (-1.2669343f +
    //               (0.27515199f -
    //                 0.25691327e-1f * x) * x) * x) * x) * x) * x;
    //
    // error 0.0000018516, which is better than 18 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbcd2769e, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3e8ce0b9, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                             getF32Constant(DAG, 0x3fa22ae7, dl));
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
    SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
                             getF32Constant(DAG, 0x40525723, dl));
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
    SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
                             getF32Constant(DAG, 0x40aaf200, dl));
    SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
    SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
                             getF32Constant(DAG, 0x40c39dad, dl));
    SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
    Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
                                 getF32Constant(DAG, 0x4042902c, dl));
  }

  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
}

// No special expansion.
return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5313}

5315/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5316/// limited-precision mode.
5317static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
                         const TargetLowering &TLI, SDNodeFlags Flags) {
// TODO: What fast-math-flags should be set on the floating-point nodes?

if (Op.getValueType() == MVT::f32 &&
    LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
  SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);

  // Scale the exponent by log10(2) [0.30102999f].
  SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
  SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
                                      getF32Constant(DAG, 0x3e9a209a, dl));

  // Get the significand and build it into a floating-point number with
  // exponent of 1.
  SDValue X = GetSignificand(DAG, Op1, dl);

  SDValue Log10ofMantissa;
  if (LimitFloatPrecision <= 6) {
    // For floating-point precision of 6:
    //
    //   Log10ofMantissa =
    //     -0.50419619f +
    //       (0.60948995f - 0.10380950f * x) * x;
    //
    // error 0.0014886165, which is 6 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0xbdd49a13, dl));
    SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3f1c0789, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
                                  getF32Constant(DAG, 0x3f011300, dl));
  } else if (LimitFloatPrecision <= 12) {
    // For floating-point precision of 12:
    //
    //   Log10ofMantissa =
    //     -0.64831180f +
    //       (0.91751397f +
    //         (-0.31664806f + 0.47637168e-1f * x) * x) * x;
    //
    // error 0.00019228036, which is better than 12 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0x3d431f31, dl));
    SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3ea21fb2, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
                             getF32Constant(DAG, 0x3f6ae232, dl));
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
    Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
                                  getF32Constant(DAG, 0x3f25f7c3, dl));
  } else { // LimitFloatPrecision <= 18
    // For floating-point precision of 18:
    //
    //   Log10ofMantissa =
    //     -0.84299375f +
    //       (1.5327582f +
    //         (-1.0688956f +
    //           (0.49102474f +
    //             (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
    //
    // error 0.0000037995730, which is better than 18 bits
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
                             getF32Constant(DAG, 0x3c5d51ce, dl));
    SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
                             getF32Constant(DAG, 0x3e00685a, dl));
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
    SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
                             getF32Constant(DAG, 0x3efb6798, dl));
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
    SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
                             getF32Constant(DAG, 0x3f88d192, dl));
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
    SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
                             getF32Constant(DAG, 0x3fc4316c, dl));
    SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
    Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
                                  getF32Constant(DAG, 0x3f57ce70, dl));
  }

  return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
}

// No special expansion.
return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5403}

5405/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5406/// limited-precision mode.
5407static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
                        const TargetLowering &TLI, SDNodeFlags Flags) {
if (Op.getValueType() == MVT::f32 &&
    LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
  return getLimitedPrecisionExp2(Op, dl, DAG);

// No special expansion.
return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5415}

5417/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5418/// limited-precision mode with x == 10.0f.
5419static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
                       SelectionDAG &DAG, const TargetLowering &TLI,
                       SDNodeFlags Flags) {
bool IsExp10 = false;
if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
    LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
  if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
    APFloat Ten(10.0f);
    IsExp10 = LHSC->isExactlyValue(Ten);
  }
}

// TODO: What fast-math-flags should be set on the FMUL node?
if (IsExp10) {
  // Put the exponent in the right bit position for later addition to the
  // final result:
  //
  //   #define LOG2OF10 3.3219281f
  //   t0 = Op * LOG2OF10;
  SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
                           getF32Constant(DAG, 0x40549a78, dl));
  return getLimitedPrecisionExp2(t0, dl, DAG);
}

// No special expansion.
return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5445}

5447/// ExpandPowI - Expand a llvm.powi intrinsic.
5448static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
                        SelectionDAG &DAG) {
// If RHS is a constant, we can expand this out to a multiplication tree if
// it's beneficial on the target, otherwise we end up lowering to a call to
// __powidf2 (for example).
if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
  unsigned Val = RHSC->getSExtValue();

  // powi(x, 0) -> 1.0
  if (Val == 0)
    return DAG.getConstantFP(1.0, DL, LHS.getValueType());

  if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI(
          Val, DAG.shouldOptForSize())) {
    // Get the exponent as a positive value.
    if ((int)Val < 0)
      Val = -Val;
    // We use the simple binary decomposition method to generate the multiply
    // sequence.  There are more optimal ways to do this (for example,
    // powi(x,15) generates one more multiply than it should), but this has
    // the benefit of being both really simple and much better than a libcall.
    SDValue Res; // Logically starts equal to 1.0
    SDValue CurSquare = LHS;
    // TODO: Intrinsics should have fast-math-flags that propagate to these
    // nodes.
    while (Val) {
      if (Val & 1) {
        if (Res.getNode())
          Res =
              DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
        else
          Res = CurSquare; // 1.0*CurSquare.
      }

      CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
                              CurSquare, CurSquare);
      Val >>= 1;
    }

    // If the original was negative, invert the result, producing 1/(x*x*x).
    if (RHSC->getSExtValue() < 0)
      Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
                        DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
    return Res;
  }
}

// Otherwise, expand to a libcall.
return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5497}

5499static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
                          SDValue LHS, SDValue RHS, SDValue Scale,
                          SelectionDAG &DAG, const TargetLowering &TLI) {
EVT VT = LHS.getValueType();
bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
LLVMContext &Ctx = *DAG.getContext();

// If the type is legal but the operation isn't, this node might survive all
// the way to operation legalization. If we end up there and we do not have
// the ability to widen the type (if VT*2 is not legal), we cannot expand the
// node.

// Coax the legalizer into expanding the node during type legalization instead
// by bumping the size by one bit. This will force it to Promote, enabling the
// early expansion and avoiding the need to expand later.

// We don't have to do this if Scale is 0; that can always be expanded, unless
// it's a saturating signed operation. Those can experience true integer
// division overflow, a case which we must avoid.

// FIXME: We wouldn't have to do this (or any of the early
// expansion/promotion) if it was possible to expand a libcall of an
// illegal type during operation legalization. But it's not, so things
// get a bit hacky.
unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue();
if ((ScaleInt > 0 || (Saturating && Signed)) &&
    (TLI.isTypeLegal(VT) ||
     (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
  TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction(
      Opcode, VT, ScaleInt);
  if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
    EVT PromVT;
    if (VT.isScalarInteger())
      PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
    else if (VT.isVector()) {
      PromVT = VT.getVectorElementType();
      PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
      PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
    } else
      llvm_unreachable("Wrong VT for DIVFIX?")::llvm::llvm_unreachable_internal("Wrong VT for DIVFIX?", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5539);
    if (Signed) {
      LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT);
      RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT);
    } else {
      LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT);
      RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT);
    }
    EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
    // For saturating operations, we need to shift up the LHS to get the
    // proper saturation width, and then shift down again afterwards.
    if (Saturating)
      LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
                        DAG.getConstant(1, DL, ShiftTy));
    SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
    if (Saturating)
      Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
                        DAG.getConstant(1, DL, ShiftTy));
    return DAG.getZExtOrTrunc(Res, DL, VT);
  }
}

return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5562}

5564// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5565// bitcasted, or split argument. Returns a list of <Register, size in bits>
5566static void
5567getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
                   const SDValue &N) {
switch (N.getOpcode()) {
case ISD::CopyFromReg: {
  SDValue Op = N.getOperand(1);
  Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
                    Op.getValueType().getSizeInBits());
  return;
}
case ISD::BITCAST:
case ISD::AssertZext:
case ISD::AssertSext:
case ISD::TRUNCATE:
  getUnderlyingArgRegs(Regs, N.getOperand(0));
  return;
case ISD::BUILD_PAIR:
case ISD::BUILD_VECTOR:
case ISD::CONCAT_VECTORS:
  for (SDValue Op : N->op_values())
    getUnderlyingArgRegs(Regs, Op);
  return;
default:
  return;
}
5591}

5593/// If the DbgValueInst is a dbg_value of a function argument, create the
5594/// corresponding DBG_VALUE machine instruction for it now.  At the end of
5595/// instruction selection, they will be inserted to the entry BB.
5596/// We don't currently support this for variadic dbg_values, as they shouldn't
5597/// appear for function arguments or in the prologue.
5598bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
  const Value *V, DILocalVariable *Variable, DIExpression *Expr,
  DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
const Argument *Arg = dyn_cast<Argument>(V);
if (!Arg)
  return false;

MachineFunction &MF = DAG.getMachineFunction();
const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

// Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
// we've been asked to pursue.
auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
                            bool Indirect) {
  if (Reg.isVirtual() && MF.useDebugInstrRef()) {
    // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
    // pointing at the VReg, which will be patched up later.
    auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
    SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg(
        /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
        /* isKill */ false, /* isDead */ false,
        /* isUndef */ false, /* isEarlyClobber */ false,
        /* SubReg */ 0, /* isDebug */ true)});

    auto *NewDIExpr = FragExpr;
    // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
    // the DIExpression.
    if (Indirect)
      NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
    SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0});
    NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
    return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
  } else {
    // Create a completely standard DBG_VALUE.
    auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
    return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
  }
};

if (Kind == FuncArgumentDbgValueKind::Value) {
  // ArgDbgValues are hoisted to the beginning of the entry block. So we
  // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
  // the entry block.
  bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
  if (!IsInEntryBlock)
    return false;

  // ArgDbgValues are hoisted to the beginning of the entry block.  So we
  // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
  // variable that also is a param.
  //
  // Although, if we are at the top of the entry block already, we can still
  // emit using ArgDbgValue. This might catch some situations when the
  // dbg.value refers to an argument that isn't used in the entry block, so
  // any CopyToReg node would be optimized out and the only way to express
  // this DBG_VALUE is by using the physical reg (or FI) as done in this
  // method.  ArgDbgValues are hoisted to the beginning of the entry block. So
  // we should only emit as ArgDbgValue if the Variable is an argument to the
  // current function, and the dbg.value intrinsic is found in the entry
  // block.
  bool VariableIsFunctionInputArg = Variable->isParameter() &&
      !DL->getInlinedAt();
  bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
  if (!IsInPrologue && !VariableIsFunctionInputArg)
    return false;

  // Here we assume that a function argument on IR level only can be used to
  // describe one input parameter on source level. If we for example have
  // source code like this
  //
  //    struct A { long x, y; };
  //    void foo(struct A a, long b) {
  //      ...
  //      b = a.x;
  //      ...
  //    }
  //
  // and IR like this
  //
  //  define void @foo(i32 %a1, i32 %a2, i32 %b)  {
  //  entry:
  //    call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
  //    call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
  //    call void @llvm.dbg.value(metadata i32 %b, "b",
  //    ...
  //    call void @llvm.dbg.value(metadata i32 %a1, "b"
  //    ...
  //
  // then the last dbg.value is describing a parameter "b" using a value that
  // is an argument. But since we already has used %a1 to describe a parameter
  // we should not handle that last dbg.value here (that would result in an
  // incorrect hoisting of the DBG_VALUE to the function entry).
  // Notice that we allow one dbg.value per IR level argument, to accommodate
  // for the situation with fragments above.
  if (VariableIsFunctionInputArg) {
    unsigned ArgNo = Arg->getArgNo();
    if (ArgNo >= FuncInfo.DescribedArgs.size())
      FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
    else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
      return false;
    FuncInfo.DescribedArgs.set(ArgNo);
  }
}

bool IsIndirect = false;
std::optional<MachineOperand> Op;
// Some arguments' frame index is recorded during argument lowering.
int FI = FuncInfo.getArgumentFrameIndex(Arg);
if (FI != std::numeric_limits<int>::max())
  Op = MachineOperand::CreateFI(FI);

SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
if (!Op && N.getNode()) {
  getUnderlyingArgRegs(ArgRegsAndSizes, N);
  Register Reg;
  if (ArgRegsAndSizes.size() == 1)
    Reg = ArgRegsAndSizes.front().first;

  if (Reg && Reg.isVirtual()) {
    MachineRegisterInfo &RegInfo = MF.getRegInfo();
    Register PR = RegInfo.getLiveInPhysReg(Reg);
    if (PR)
      Reg = PR;
  }
  if (Reg) {
    Op = MachineOperand::CreateReg(Reg, false);
    IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
  }
}

if (!Op && N.getNode()) {
  // Check if frame index is available.
  SDValue LCandidate = peekThroughBitcasts(N);
  if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
    if (FrameIndexSDNode *FINode =
        dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
      Op = MachineOperand::CreateFI(FINode->getIndex());
}

if (!Op) {
  // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
  auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
                                       SplitRegs) {
    unsigned Offset = 0;
    for (const auto &RegAndSize : SplitRegs) {
      // If the expression is already a fragment, the current register
      // offset+size might extend beyond the fragment. In this case, only
      // the register bits that are inside the fragment are relevant.
      int RegFragmentSizeInBits = RegAndSize.second;
      if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
        uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
        // The register is entirely outside the expression fragment,
        // so is irrelevant for debug info.
        if (Offset >= ExprFragmentSizeInBits)
          break;
        // The register is partially outside the expression fragment, only
        // the low bits within the fragment are relevant for debug info.
        if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
          RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
        }
      }

      auto FragmentExpr = DIExpression::createFragmentExpression(
          Expr, Offset, RegFragmentSizeInBits);
      Offset += RegAndSize.second;
      // If a valid fragment expression cannot be created, the variable's
      // correct value cannot be determined and so it is set as Undef.
      if (!FragmentExpr) {
        SDDbgValue *SDV = DAG.getConstantDbgValue(
            Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
        DAG.AddDbgValue(SDV, false);
        continue;
      }
      MachineInstr *NewMI =
          MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
                           Kind != FuncArgumentDbgValueKind::Value);
      FuncInfo.ArgDbgValues.push_back(NewMI);
    }
  };

  // Check if ValueMap has reg number.
  DenseMap<const Value *, Register>::const_iterator
    VMI = FuncInfo.ValueMap.find(V);
  if (VMI != FuncInfo.ValueMap.end()) {
    const auto &TLI = DAG.getTargetLoweringInfo();
    RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
                     V->getType(), std::nullopt);
    if (RFV.occupiesMultipleRegs()) {
      splitMultiRegDbgValue(RFV.getRegsAndSizes());
      return true;
    }

    Op = MachineOperand::CreateReg(VMI->second, false);
    IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
  } else if (ArgRegsAndSizes.size() > 1) {
    // This was split due to the calling convention, and no virtual register
    // mapping exists for the value.
    splitMultiRegDbgValue(ArgRegsAndSizes);
    return true;
  }
}

if (!Op)
  return false;

assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
 (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5804
, __extension__ __PRETTY_FUNCTION__))
       "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
 (0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5804
, __extension__ __PRETTY_FUNCTION__));
MachineInstr *NewMI = nullptr;

if (Op->isReg())
  NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
else
  NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
                  Variable, Expr);

// Otherwise, use ArgDbgValues.
FuncInfo.ArgDbgValues.push_back(NewMI);
return true;
5816}

5818/// Return the appropriate SDDbgValue based on N.
5819SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
                                           DILocalVariable *Variable,
                                           DIExpression *Expr,
                                           const DebugLoc &dl,
                                           unsigned DbgSDNodeOrder) {
if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
  // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
  // stack slot locations.
  //
  // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
  // debug values here after optimization:
  //
  //   dbg.value(i32* %px, !"int *px", !DIExpression()), and
  //   dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
  //
  // Both describe the direct values of their associated variables.
  return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
                                   /*IsIndirect*/ false, dl, DbgSDNodeOrder);
}
return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
                       /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5840}

5842static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
switch (Intrinsic) {
case Intrinsic::smul_fix:
  return ISD::SMULFIX;
case Intrinsic::umul_fix:
  return ISD::UMULFIX;
case Intrinsic::smul_fix_sat:
  return ISD::SMULFIXSAT;
case Intrinsic::umul_fix_sat:
  return ISD::UMULFIXSAT;
case Intrinsic::sdiv_fix:
  return ISD::SDIVFIX;
case Intrinsic::udiv_fix:
  return ISD::UDIVFIX;
case Intrinsic::sdiv_fix_sat:
  return ISD::SDIVFIXSAT;
case Intrinsic::udiv_fix_sat:
  return ISD::UDIVFIXSAT;
default:
  llvm_unreachable("Unhandled fixed point intrinsic")::llvm::llvm_unreachable_internal("Unhandled fixed point intrinsic"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5861
);
}
5863}

5865void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
                                         const char *FunctionName) {
assert(FunctionName && "FunctionName must not be nullptr")(static_cast <bool> (FunctionName && "FunctionName must not be nullptr"
) ? void (0) : __assert_fail ("FunctionName && \"FunctionName must not be nullptr\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5867
, __extension__ __PRETTY_FUNCTION__));
SDValue Callee = DAG.getExternalSymbol(
    FunctionName,
    DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
5872}

5874/// Given a @llvm.call.preallocated.setup, return the corresponding
5875/// preallocated call.
5876static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
assert(cast<CallBase>(PreallocatedSetup)(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880
, __extension__ __PRETTY_FUNCTION__))
               ->getCalledFunction()(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880
, __extension__ __PRETTY_FUNCTION__))
               ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880
, __extension__ __PRETTY_FUNCTION__))
       "expected call_preallocated_setup Value")(static_cast <bool> (cast<CallBase>(PreallocatedSetup
) ->getCalledFunction() ->getIntrinsicID() == Intrinsic
::call_preallocated_setup && "expected call_preallocated_setup Value"
) ? void (0) : __assert_fail ("cast<CallBase>(PreallocatedSetup) ->getCalledFunction() ->getIntrinsicID() == Intrinsic::call_preallocated_setup && \"expected call_preallocated_setup Value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5880
, __extension__ __PRETTY_FUNCTION__));
for (const auto *U : PreallocatedSetup->users()) {
  auto *UseCall = cast<CallBase>(U);
  const Function *Fn = UseCall->getCalledFunction();
  if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
    return UseCall;
  }
}
llvm_unreachable("expected corresponding call to preallocated setup/arg")::llvm::llvm_unreachable_internal("expected corresponding call to preallocated setup/arg"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5888
);
5889}

5891/// Lower the call to the specified intrinsic function.
5892void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                                           unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc sdl = getCurSDLoc();
DebugLoc dl = getCurDebugLoc();
SDValue Res;

SDNodeFlags Flags;
if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
  Flags.copyFMF(*FPOp);

switch (Intrinsic) {
default:
  // By default, turn this into a target intrinsic node.
  visitTargetIntrinsic(I, Intrinsic);
  return;
case Intrinsic::vscale: {
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
  return;
}
case Intrinsic::vastart:  visitVAStart(I); return;
case Intrinsic::vaend:    visitVAEnd(I); return;
case Intrinsic::vacopy:   visitVACopy(I); return;
case Intrinsic::returnaddress:
  setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
                           TLI.getValueType(DAG.getDataLayout(), I.getType()),
                           getValue(I.getArgOperand(0))));
  return;
case Intrinsic::addressofreturnaddress:
  setValue(&I,
           DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
                       TLI.getValueType(DAG.getDataLayout(), I.getType())));
  return;
case Intrinsic::sponentry:
  setValue(&I,
           DAG.getNode(ISD::SPONENTRY, sdl,
                       TLI.getValueType(DAG.getDataLayout(), I.getType())));
  return;
case Intrinsic::frameaddress:
  setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
                           TLI.getFrameIndexTy(DAG.getDataLayout()),
                           getValue(I.getArgOperand(0))));
  return;
case Intrinsic::read_volatile_register:
case Intrinsic::read_register: {
  Value *Reg = I.getArgOperand(0);
  SDValue Chain = getRoot();
  SDValue RegName =
      DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  Res = DAG.getNode(ISD::READ_REGISTER, sdl,
    DAG.getVTList(VT, MVT::Other), Chain, RegName);
  setValue(&I, Res);
  DAG.setRoot(Res.getValue(1));
  return;
}
case Intrinsic::write_register: {
  Value *Reg = I.getArgOperand(0);
  Value *RegValue = I.getArgOperand(1);
  SDValue Chain = getRoot();
  SDValue RegName =
      DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
  DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
                          RegName, getValue(RegValue)));
  return;
}
case Intrinsic::memcpy: {
  const auto &MCI = cast<MemCpyInst>(I);
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  SDValue Op3 = getValue(I.getArgOperand(2));
  // @llvm.memcpy defines 0 and 1 to both mean no alignment.
  Align DstAlign = MCI.getDestAlign().valueOrOne();
  Align SrcAlign = MCI.getSourceAlign().valueOrOne();
  Align Alignment = std::min(DstAlign, SrcAlign);
  bool isVol = MCI.isVolatile();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  // FIXME: Support passing different dest/src alignments to the memcpy DAG
  // node.
  SDValue Root = isVol ? getRoot() : getMemoryRoot();
  SDValue MC = DAG.getMemcpy(
      Root, sdl, Op1, Op2, Op3, Alignment, isVol,
      /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)),
      MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
  updateDAGForMaybeTailCall(MC);
  return;
}
case Intrinsic::memcpy_inline: {
  const auto &MCI = cast<MemCpyInlineInst>(I);
  SDValue Dst = getValue(I.getArgOperand(0));
  SDValue Src = getValue(I.getArgOperand(1));
  SDValue Size = getValue(I.getArgOperand(2));
  assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size")(static_cast <bool> (isa<ConstantSDNode>(Size) &&
 "memcpy_inline needs constant size") ? void (0) : __assert_fail
 ("isa<ConstantSDNode>(Size) && \"memcpy_inline needs constant size\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 5985
, __extension__ __PRETTY_FUNCTION__));
  // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
  Align DstAlign = MCI.getDestAlign().valueOrOne();
  Align SrcAlign = MCI.getSourceAlign().valueOrOne();
  Align Alignment = std::min(DstAlign, SrcAlign);
  bool isVol = MCI.isVolatile();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  // FIXME: Support passing different dest/src alignments to the memcpy DAG
  // node.
  SDValue MC = DAG.getMemcpy(
      getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
      /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)),
      MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA);
  updateDAGForMaybeTailCall(MC);
  return;
}
case Intrinsic::memset: {
  const auto &MSI = cast<MemSetInst>(I);
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  SDValue Op3 = getValue(I.getArgOperand(2));
  // @llvm.memset defines 0 and 1 to both mean no alignment.
  Align Alignment = MSI.getDestAlign().valueOrOne();
  bool isVol = MSI.isVolatile();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  SDValue Root = isVol ? getRoot() : getMemoryRoot();
  SDValue MS = DAG.getMemset(
      Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
      isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
  updateDAGForMaybeTailCall(MS);
  return;
}
case Intrinsic::memset_inline: {
  const auto &MSII = cast<MemSetInlineInst>(I);
  SDValue Dst = getValue(I.getArgOperand(0));
  SDValue Value = getValue(I.getArgOperand(1));
  SDValue Size = getValue(I.getArgOperand(2));
  assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size")(static_cast <bool> (isa<ConstantSDNode>(Size) &&
 "memset_inline needs constant size") ? void (0) : __assert_fail
 ("isa<ConstantSDNode>(Size) && \"memset_inline needs constant size\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6022
, __extension__ __PRETTY_FUNCTION__));
  // @llvm.memset defines 0 and 1 to both mean no alignment.
  Align DstAlign = MSII.getDestAlign().valueOrOne();
  bool isVol = MSII.isVolatile();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  SDValue Root = isVol ? getRoot() : getMemoryRoot();
  SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
                             /* AlwaysInline */ true, isTC,
                             MachinePointerInfo(I.getArgOperand(0)),
                             I.getAAMetadata());
  updateDAGForMaybeTailCall(MC);
  return;
}
case Intrinsic::memmove: {
  const auto &MMI = cast<MemMoveInst>(I);
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  SDValue Op3 = getValue(I.getArgOperand(2));
  // @llvm.memmove defines 0 and 1 to both mean no alignment.
  Align DstAlign = MMI.getDestAlign().valueOrOne();
  Align SrcAlign = MMI.getSourceAlign().valueOrOne();
  Align Alignment = std::min(DstAlign, SrcAlign);
  bool isVol = MMI.isVolatile();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  // FIXME: Support passing different dest/src alignments to the memmove DAG
  // node.
  SDValue Root = isVol ? getRoot() : getMemoryRoot();
  SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
                              isTC, MachinePointerInfo(I.getArgOperand(0)),
                              MachinePointerInfo(I.getArgOperand(1)),
                              I.getAAMetadata(), AA);
  updateDAGForMaybeTailCall(MM);
  return;
}
case Intrinsic::memcpy_element_unordered_atomic: {
  const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
  SDValue Dst = getValue(MI.getRawDest());
  SDValue Src = getValue(MI.getRawSource());
  SDValue Length = getValue(MI.getLength());

  Type *LengthTy = MI.getLength()->getType();
  unsigned ElemSz = MI.getElementSizeInBytes();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  SDValue MC =
      DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
                          isTC, MachinePointerInfo(MI.getRawDest()),
                          MachinePointerInfo(MI.getRawSource()));
  updateDAGForMaybeTailCall(MC);
  return;
}
case Intrinsic::memmove_element_unordered_atomic: {
  auto &MI = cast<AtomicMemMoveInst>(I);
  SDValue Dst = getValue(MI.getRawDest());
  SDValue Src = getValue(MI.getRawSource());
  SDValue Length = getValue(MI.getLength());

  Type *LengthTy = MI.getLength()->getType();
  unsigned ElemSz = MI.getElementSizeInBytes();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  SDValue MC =
      DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
                           isTC, MachinePointerInfo(MI.getRawDest()),
                           MachinePointerInfo(MI.getRawSource()));
  updateDAGForMaybeTailCall(MC);
  return;
}
case Intrinsic::memset_element_unordered_atomic: {
  auto &MI = cast<AtomicMemSetInst>(I);
  SDValue Dst = getValue(MI.getRawDest());
  SDValue Val = getValue(MI.getValue());
  SDValue Length = getValue(MI.getLength());

  Type *LengthTy = MI.getLength()->getType();
  unsigned ElemSz = MI.getElementSizeInBytes();
  bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
  SDValue MC =
      DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
                          isTC, MachinePointerInfo(MI.getRawDest()));
  updateDAGForMaybeTailCall(MC);
  return;
}
case Intrinsic::call_preallocated_setup: {
  const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
  SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
  SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
                            getRoot(), SrcValue);
  setValue(&I, Res);
  DAG.setRoot(Res);
  return;
}
case Intrinsic::call_preallocated_arg: {
  const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
  SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
  SDValue Ops[3];
  Ops[0] = getRoot();
  Ops[1] = SrcValue;
  Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
                                 MVT::i32); // arg index
  SDValue Res = DAG.getNode(
      ISD::PREALLOCATED_ARG, sdl,
      DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
  setValue(&I, Res);
  DAG.setRoot(Res.getValue(1));
  return;
}
case Intrinsic::dbg_declare: {
  const auto &DI = cast<DbgDeclareInst>(I);
  // Debug intrinsics are handled separately in assignment tracking mode.
  // Some intrinsics are handled right after Argument lowering.
  if (AssignmentTrackingEnabled ||
      FuncInfo.PreprocessedDbgDeclares.count(&DI))
    return;
  // Assume dbg.declare can not currently use DIArgList, i.e.
  // it is non-variadic.
  assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList")(static_cast <bool> (!DI.hasArgList() && "Only dbg.value should currently use DIArgList"
) ? void (0) : __assert_fail ("!DI.hasArgList() && \"Only dbg.value should currently use DIArgList\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6136
, __extension__ __PRETTY_FUNCTION__));
  DILocalVariable *Variable = DI.getVariable();
  DIExpression *Expression = DI.getExpression();
  dropDanglingDebugInfo(Variable, Expression);
  assert(Variable && "Missing variable")(static_cast <bool> (Variable && "Missing variable"
) ? void (0) : __assert_fail ("Variable && \"Missing variable\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6140
, __extension__ __PRETTY_FUNCTION__));
  LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: "
 << DI << "\n"; } } while (false)
                    << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "SelectionDAG visiting debug intrinsic: "
 << DI << "\n"; } } while (false);
  // Check if address has undef value.
  const Value *Address = DI.getVariableLocationOp(0);
  if (!Address || isa<UndefValue>(Address) ||
      (Address->use_empty() && !isa<Argument>(Address))) {
    LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
 DI << " (bad/undef/unused-arg address)\n"; } } while (
false)
                      << " (bad/undef/unused-arg address)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
 DI << " (bad/undef/unused-arg address)\n"; } } while (
false);
    return;
  }

  bool isParameter = Variable->isParameter() || isa<Argument>(Address);

  SDValue &N = NodeMap[Address];
  if (!N.getNode() && isa<Argument>(Address))
    // Check unused arguments map.
    N = UnusedArgNodeMap[Address];
  SDDbgValue *SDV;
  if (N.getNode()) {
    if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
      Address = BCI->getOperand(0);
    // Parameters are handled specially.
    auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
    if (isParameter && FINode) {
      // Byval parameter. We have a frame index at this point.
      SDV =
          DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
                                    /*IsIndirect*/ true, dl, SDNodeOrder);
    } else if (isa<Argument>(Address)) {
      // Address is an argument, so try to emit its dbg value using
      // virtual register info from the FuncInfo.ValueMap.
      EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
                               FuncArgumentDbgValueKind::Declare, N);
      return;
    } else {
      SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
                            true, dl, SDNodeOrder);
    }
    DAG.AddDbgValue(SDV, isParameter);
  } else {
    // If Address is an argument then try to emit its dbg value using
    // virtual register info from the FuncInfo.ValueMap.
    if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl,
                                  FuncArgumentDbgValueKind::Declare, N)) {
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
 DI << " (could not emit func-arg dbg_value)\n"; } } while
 (false)
                        << " (could not emit func-arg dbg_value)\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
 DI << " (could not emit func-arg dbg_value)\n"; } } while
 (false);
    }
  }
  return;
}
case Intrinsic::dbg_label: {
  const DbgLabelInst &DI = cast<DbgLabelInst>(I);
  DILabel *Label = DI.getLabel();
  assert(Label && "Missing label")(static_cast <bool> (Label && "Missing label") ?
 void (0) : __assert_fail ("Label && \"Missing label\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6194
, __extension__ __PRETTY_FUNCTION__));

  SDDbgLabel *SDV;
  SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
  DAG.AddDbgLabel(SDV);
  return;
}
case Intrinsic::dbg_assign: {
  // Debug intrinsics are handled seperately in assignment tracking mode.
  if (AssignmentTrackingEnabled)
    return;
  // If assignment tracking hasn't been enabled then fall through and treat
  // the dbg.assign as a dbg.value.
  [[fallthrough]];
}
case Intrinsic::dbg_value: {
  // Debug intrinsics are handled seperately in assignment tracking mode.
  if (AssignmentTrackingEnabled)
    return;
  const DbgValueInst &DI = cast<DbgValueInst>(I);
  assert(DI.getVariable() && "Missing variable")(static_cast <bool> (DI.getVariable() && "Missing variable"
) ? void (0) : __assert_fail ("DI.getVariable() && \"Missing variable\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6214
, __extension__ __PRETTY_FUNCTION__));

  DILocalVariable *Variable = DI.getVariable();
  DIExpression *Expression = DI.getExpression();
  dropDanglingDebugInfo(Variable, Expression);

  if (DI.isKillLocation()) {
    handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
    return;
  }

  SmallVector<Value *, 4> Values(DI.getValues());
  if (Values.empty())
    return;

  bool IsVariadic = DI.hasArgList();
  if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
                        SDNodeOrder, IsVariadic))
    addDanglingDebugInfo(&DI, SDNodeOrder);
  return;
}

case Intrinsic::eh_typeid_for: {
  // Find the type id for the given typeinfo.
  GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
  unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
  Res = DAG.getConstant(TypeID, sdl, MVT::i32);
  setValue(&I, Res);
  return;
}

case Intrinsic::eh_return_i32:
case Intrinsic::eh_return_i64:
  DAG.getMachineFunction().setCallsEHReturn(true);
  DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
                          MVT::Other,
                          getControlRoot(),
                          getValue(I.getArgOperand(0)),
                          getValue(I.getArgOperand(1))));
  return;
case Intrinsic::eh_unwind_init:
  DAG.getMachineFunction().setCallsUnwindInit(true);
  return;
case Intrinsic::eh_dwarf_cfa:
  setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
                           TLI.getPointerTy(DAG.getDataLayout()),
                           getValue(I.getArgOperand(0))));
  return;
case Intrinsic::eh_sjlj_callsite: {
  MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
  assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")(static_cast <bool> (MMI.getCurrentCallSite() == 0 &&
 "Overlapping call sites!") ? void (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6265
, __extension__ __PRETTY_FUNCTION__));

  MMI.setCurrentCallSite(CI->getZExtValue());
  return;
}
case Intrinsic::eh_sjlj_functioncontext: {
  // Get and store the index of the function context.
  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
  AllocaInst *FnCtx =
    cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
  int FI = FuncInfo.StaticAllocaMap[FnCtx];
  MFI.setFunctionContextIndex(FI);
  return;
}
case Intrinsic::eh_sjlj_setjmp: {
  SDValue Ops[2];
  Ops[0] = getRoot();
  Ops[1] = getValue(I.getArgOperand(0));
  SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
                           DAG.getVTList(MVT::i32, MVT::Other), Ops);
  setValue(&I, Op.getValue(0));
  DAG.setRoot(Op.getValue(1));
  return;
}
case Intrinsic::eh_sjlj_longjmp:
  DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
                          getRoot(), getValue(I.getArgOperand(0))));
  return;
case Intrinsic::eh_sjlj_setup_dispatch:
  DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
                          getRoot()));
  return;
case Intrinsic::masked_gather:
  visitMaskedGather(I);
  return;
case Intrinsic::masked_load:
  visitMaskedLoad(I);
  return;
case Intrinsic::masked_scatter:
  visitMaskedScatter(I);
  return;
case Intrinsic::masked_store:
  visitMaskedStore(I);
  return;
case Intrinsic::masked_expandload:
  visitMaskedLoad(I, true /* IsExpanding */);
  return;
case Intrinsic::masked_compressstore:
  visitMaskedStore(I, true /* IsCompressing */);
  return;
case Intrinsic::powi:
  setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
                          getValue(I.getArgOperand(1)), DAG));
  return;
case Intrinsic::log:
  setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
  return;
case Intrinsic::log2:
  setValue(&I,
           expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
  return;
case Intrinsic::log10:
  setValue(&I,
           expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
  return;
case Intrinsic::exp:
  setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
  return;
case Intrinsic::exp2:
  setValue(&I,
           expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
  return;
case Intrinsic::pow:
  setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
                         getValue(I.getArgOperand(1)), DAG, TLI, Flags));
  return;
case Intrinsic::sqrt:
case Intrinsic::fabs:
case Intrinsic::sin:
case Intrinsic::cos:
case Intrinsic::floor:
case Intrinsic::ceil:
case Intrinsic::trunc:
case Intrinsic::rint:
case Intrinsic::nearbyint:
case Intrinsic::round:
case Intrinsic::roundeven:
case Intrinsic::canonicalize: {
  unsigned Opcode;
  switch (Intrinsic) {
  default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6355);  // Can't reach here.
  case Intrinsic::sqrt:      Opcode = ISD::FSQRT;      break;
  case Intrinsic::fabs:      Opcode = ISD::FABS;       break;
  case Intrinsic::sin:       Opcode = ISD::FSIN;       break;
  case Intrinsic::cos:       Opcode = ISD::FCOS;       break;
  case Intrinsic::floor:     Opcode = ISD::FFLOOR;     break;
  case Intrinsic::ceil:      Opcode = ISD::FCEIL;      break;
  case Intrinsic::trunc:     Opcode = ISD::FTRUNC;     break;
  case Intrinsic::rint:      Opcode = ISD::FRINT;      break;
  case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
  case Intrinsic::round:     Opcode = ISD::FROUND;     break;
  case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
  case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
  }

  setValue(&I, DAG.getNode(Opcode, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)), Flags));
  return;
}
case Intrinsic::lround:
case Intrinsic::llround:
case Intrinsic::lrint:
case Intrinsic::llrint: {
  unsigned Opcode;
  switch (Intrinsic) {
  default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6381);  // Can't reach here.
  case Intrinsic::lround:  Opcode = ISD::LROUND;  break;
  case Intrinsic::llround: Opcode = ISD::LLROUND; break;
  case Intrinsic::lrint:   Opcode = ISD::LRINT;   break;
  case Intrinsic::llrint:  Opcode = ISD::LLRINT;  break;
  }

  EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
                           getValue(I.getArgOperand(0))));
  return;
}
case Intrinsic::minnum:
  setValue(&I, DAG.getNode(ISD::FMINNUM, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)),
                           getValue(I.getArgOperand(1)), Flags));
  return;
case Intrinsic::maxnum:
  setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)),
                           getValue(I.getArgOperand(1)), Flags));
  return;
case Intrinsic::minimum:
  setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)),
                           getValue(I.getArgOperand(1)), Flags));
  return;
case Intrinsic::maximum:
  setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)),
                           getValue(I.getArgOperand(1)), Flags));
  return;
case Intrinsic::copysign:
  setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)),
                           getValue(I.getArgOperand(1)), Flags));
  return;
case Intrinsic::arithmetic_fence: {
  setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0)), Flags));
  return;
}
case Intrinsic::fma:
  setValue(&I, DAG.getNode(
                   ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
                   getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
                   getValue(I.getArgOperand(2)), Flags));
  return;
6435#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
case Intrinsic::INTRINSIC:
6437#include "llvm/IR/ConstrainedOps.def"
  visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
  return;
6440#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
6441#include "llvm/IR/VPIntrinsics.def"
  visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
  return;
case Intrinsic::fptrunc_round: {
  // Get the last argument, the metadata and convert it to an integer in the
  // call
  Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
  std::optional<RoundingMode> RoundMode =
      convertStrToRoundingMode(cast<MDString>(MD)->getString());

  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

  // Propagate fast-math-flags from IR to node(s).
  SDNodeFlags Flags;
  Flags.copyFMF(*cast<FPMathOperator>(&I));
  SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);

  SDValue Result;
  Result = DAG.getNode(
      ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
      DAG.getTargetConstant((int)*RoundMode, sdl,
                            TLI.getPointerTy(DAG.getDataLayout())));
  setValue(&I, Result);

  return;
}
case Intrinsic::fmuladd: {
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
      TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
    setValue(&I, DAG.getNode(ISD::FMA, sdl,
                             getValue(I.getArgOperand(0)).getValueType(),
                             getValue(I.getArgOperand(0)),
                             getValue(I.getArgOperand(1)),
                             getValue(I.getArgOperand(2)), Flags));
  } else {
    // TODO: Intrinsic calls should have fast-math-flags.
    SDValue Mul = DAG.getNode(
        ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
        getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
    SDValue Add = DAG.getNode(ISD::FADD, sdl,
                              getValue(I.getArgOperand(0)).getValueType(),
                              Mul, getValue(I.getArgOperand(2)), Flags);
    setValue(&I, Add);
  }
  return;
}
case Intrinsic::convert_to_fp16:
  setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
                           DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
                                       getValue(I.getArgOperand(0)),
                                       DAG.getTargetConstant(0, sdl,
                                                             MVT::i32))));
  return;
case Intrinsic::convert_from_fp16:
  setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
                           TLI.getValueType(DAG.getDataLayout(), I.getType()),
                           DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
                                       getValue(I.getArgOperand(0)))));
  return;
case Intrinsic::fptosi_sat: {
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT,
                           getValue(I.getArgOperand(0)),
                           DAG.getValueType(VT.getScalarType())));
  return;
}
case Intrinsic::fptoui_sat: {
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT,
                           getValue(I.getArgOperand(0)),
                           DAG.getValueType(VT.getScalarType())));
  return;
}
case Intrinsic::set_rounding:
  Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
                    {getRoot(), getValue(I.getArgOperand(0))});
  setValue(&I, Res);
  DAG.setRoot(Res.getValue(0));
  return;
case Intrinsic::is_fpclass: {
  const DataLayout DLayout = DAG.getDataLayout();
  EVT DestVT = TLI.getValueType(DLayout, I.getType());
  EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
  FPClassTest Test = static_cast<FPClassTest>(
      cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &F = MF.getFunction();
  SDValue Op = getValue(I.getArgOperand(0));
  SDNodeFlags Flags;
  Flags.setNoFPExcept(
      !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
  // If ISD::IS_FPCLASS should be expanded, do it right now, because the
  // expansion can use illegal types. Making expansion early allows
  // legalizing these types prior to selection.
  if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) {
    SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
    setValue(&I, Result);
    return;
  }

  SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
  SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
  setValue(&I, V);
  return;
}
case Intrinsic::pcmarker: {
  SDValue Tmp = getValue(I.getArgOperand(0));
  DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
  return;
}
case Intrinsic::readcyclecounter: {
  SDValue Op = getRoot();
  Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
                    DAG.getVTList(MVT::i64, MVT::Other), Op);
  setValue(&I, Res);
  DAG.setRoot(Res.getValue(1));
  return;
}
case Intrinsic::bitreverse:
  setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0))));
  return;
case Intrinsic::bswap:
  setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
                           getValue(I.getArgOperand(0)).getValueType(),
                           getValue(I.getArgOperand(0))));
  return;
case Intrinsic::cttz: {
  SDValue Arg = getValue(I.getArgOperand(0));
  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
  EVT Ty = Arg.getValueType();
  setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
                           sdl, Ty, Arg));
  return;
}
case Intrinsic::ctlz: {
  SDValue Arg = getValue(I.getArgOperand(0));
  ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
  EVT Ty = Arg.getValueType();
  setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
                           sdl, Ty, Arg));
  return;
}
case Intrinsic::ctpop: {
  SDValue Arg = getValue(I.getArgOperand(0));
  EVT Ty = Arg.getValueType();
  setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
  return;
}
case Intrinsic::fshl:
case Intrinsic::fshr: {
  bool IsFSHL = Intrinsic == Intrinsic::fshl;
  SDValue X = getValue(I.getArgOperand(0));
  SDValue Y = getValue(I.getArgOperand(1));
  SDValue Z = getValue(I.getArgOperand(2));
  EVT VT = X.getValueType();

  if (X == Y) {
    auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
    setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
  } else {
    auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
    setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
  }
  return;
}
case Intrinsic::sadd_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::uadd_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::ssub_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::usub_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::sshl_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::ushl_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::smul_fix:
case Intrinsic::umul_fix:
case Intrinsic::smul_fix_sat:
case Intrinsic::umul_fix_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  SDValue Op3 = getValue(I.getArgOperand(2));
  setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
                           Op1.getValueType(), Op1, Op2, Op3));
  return;
}
case Intrinsic::sdiv_fix:
case Intrinsic::udiv_fix:
case Intrinsic::sdiv_fix_sat:
case Intrinsic::udiv_fix_sat: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  SDValue Op3 = getValue(I.getArgOperand(2));
  setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
                            Op1, Op2, Op3, DAG, TLI));
  return;
}
case Intrinsic::smax: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::smin: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::umax: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::umin: {
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));
  setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
  return;
}
case Intrinsic::abs: {
  // TODO: Preserve "int min is poison" arg in SDAG?
  SDValue Op1 = getValue(I.getArgOperand(0));
  setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
  return;
}
case Intrinsic::stacksave: {
  SDValue Op = getRoot();
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
  setValue(&I, Res);
  DAG.setRoot(Res.getValue(1));
  return;
}
case Intrinsic::stackrestore:
  Res = getValue(I.getArgOperand(0));
  DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
  return;
case Intrinsic::get_dynamic_area_offset: {
  SDValue Op = getRoot();
  EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
  EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
  // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
  // target.
  if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
    report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
                       " intrinsic!");
  Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
                    Op);
  DAG.setRoot(Op);
  setValue(&I, Res);
  return;
}
case Intrinsic::stackguard: {
  MachineFunction &MF = DAG.getMachineFunction();
  const Module &M = *MF.getFunction().getParent();
  SDValue Chain = getRoot();
  if (TLI.useLoadStackGuardNode()) {
    Res = getLoadStackGuard(DAG, sdl, Chain);
  } else {
    EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
    const Value *Global = TLI.getSDagStackGuard(M);
    Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType());
    Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
                      MachinePointerInfo(Global, 0), Align,
                      MachineMemOperand::MOVolatile);
  }
  if (TLI.useStackGuardXorFP())
    Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
  DAG.setRoot(Chain);
  setValue(&I, Res);
  return;
}
case Intrinsic::stackprotector: {
  // Emit code into the DAG to store the stack guard onto the stack.
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  SDValue Src, Chain = getRoot();

  if (TLI.useLoadStackGuardNode())
    Src = getLoadStackGuard(DAG, sdl, Chain);
  else
    Src = getValue(I.getArgOperand(0));   // The guard's value.

  AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));

  int FI = FuncInfo.StaticAllocaMap[Slot];
  MFI.setStackProtectorIndex(FI);
  EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());

  SDValue FIN = DAG.getFrameIndex(FI, PtrTy);

  // Store the stack protector onto the stack.
  Res = DAG.getStore(
      Chain, sdl, Src, FIN,
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
      MaybeAlign(), MachineMemOperand::MOVolatile);
  setValue(&I, Res);
  DAG.setRoot(Res);
  return;
}
case Intrinsic::objectsize:
  llvm_unreachable("llvm.objectsize.* should have been lowered already")::llvm::llvm_unreachable_internal("llvm.objectsize.* should have been lowered already"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6773
);

case Intrinsic::is_constant:
  llvm_unreachable("llvm.is.constant.* should have been lowered already")::llvm::llvm_unreachable_internal("llvm.is.constant.* should have been lowered already"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6776
);

case Intrinsic::annotation:
case Intrinsic::ptr_annotation:
case Intrinsic::launder_invariant_group:
case Intrinsic::strip_invariant_group:
  // Drop the intrinsic, but forward the value
  setValue(&I, getValue(I.getOperand(0)));
  return;

case Intrinsic::assume:
case Intrinsic::experimental_noalias_scope_decl:
case Intrinsic::var_annotation:
case Intrinsic::sideeffect:
  // Discard annotate attributes, noalias scope declarations, assumptions, and
  // artificial side-effects.
  return;

case Intrinsic::codeview_annotation: {
  // Emit a label associated with this metadata.
  MachineFunction &MF = DAG.getMachineFunction();
  MCSymbol *Label =
      MF.getMMI().getContext().createTempSymbol("annotation", true);
  Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
  MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
  Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
  DAG.setRoot(Res);
  return;
}

case Intrinsic::init_trampoline: {
  const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());

  SDValue Ops[6];
  Ops[0] = getRoot();
  Ops[1] = getValue(I.getArgOperand(0));
  Ops[2] = getValue(I.getArgOperand(1));
  Ops[3] = getValue(I.getArgOperand(2));
  Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
  Ops[5] = DAG.getSrcValue(F);

  Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);

  DAG.setRoot(Res);
  return;
}
case Intrinsic::adjust_trampoline:
  setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
                           TLI.getPointerTy(DAG.getDataLayout()),
                           getValue(I.getArgOperand(0))));
  return;
case Intrinsic::gcroot: {
  assert(DAG.getMachineFunction().getFunction().hasGC() &&(static_cast <bool> (DAG.getMachineFunction().getFunction
().hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6829
, __extension__ __PRETTY_FUNCTION__))
         "only valid in functions with gc specified, enforced by Verifier")(static_cast <bool> (DAG.getMachineFunction().getFunction
().hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6829
, __extension__ __PRETTY_FUNCTION__));
  assert(GFI && "implied by previous")(static_cast <bool> (GFI && "implied by previous"
) ? void (0) : __assert_fail ("GFI && \"implied by previous\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6830
, __extension__ __PRETTY_FUNCTION__));
  const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
  const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));

  FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
  GFI->addStackRoot(FI->getIndex(), TypeMap);
  return;
}
case Intrinsic::gcread:
case Intrinsic::gcwrite:
  llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 6840
);
case Intrinsic::get_rounding:
  Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
  setValue(&I, Res);
  DAG.setRoot(Res.getValue(1));
  return;

case Intrinsic::expect:
  // Just replace __builtin_expect(exp, c) with EXP.
  setValue(&I, getValue(I.getArgOperand(0)));
  return;

case Intrinsic::ubsantrap:
case Intrinsic::debugtrap:
case Intrinsic::trap: {
  StringRef TrapFuncName =
      I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
  if (TrapFuncName.empty()) {
    switch (Intrinsic) {
    case Intrinsic::trap:
      DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
      break;
    case Intrinsic::debugtrap:
      DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
      break;
    case Intrinsic::ubsantrap:
      DAG.setRoot(DAG.getNode(
          ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
          DAG.getTargetConstant(
              cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
              MVT::i32)));
      break;
    default: llvm_unreachable("unknown trap intrinsic")::llvm::llvm_unreachable_internal("unknown trap intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6872);
    }
    return;
  }
  TargetLowering::ArgListTy Args;
  if (Intrinsic == Intrinsic::ubsantrap) {
    Args.push_back(TargetLoweringBase::ArgListEntry());
    Args[0].Val = I.getArgOperand(0);
    Args[0].Node = getValue(Args[0].Val);
    Args[0].Ty = Args[0].Val->getType();
  }

  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
      CallingConv::C, I.getType(),
      DAG.getExternalSymbol(TrapFuncName.data(),
                            TLI.getPointerTy(DAG.getDataLayout())),
      std::move(Args));

  std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
  DAG.setRoot(Result.second);
  return;
}

case Intrinsic::uadd_with_overflow:
case Intrinsic::sadd_with_overflow:
case Intrinsic::usub_with_overflow:
case Intrinsic::ssub_with_overflow:
case Intrinsic::umul_with_overflow:
case Intrinsic::smul_with_overflow: {
  ISD::NodeType Op;
  switch (Intrinsic) {
  default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6904);  // Can't reach here.
  case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
  case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
  case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
  case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
  case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
  case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
  }
  SDValue Op1 = getValue(I.getArgOperand(0));
  SDValue Op2 = getValue(I.getArgOperand(1));

  EVT ResultVT = Op1.getValueType();
  EVT OverflowVT = MVT::i1;
  if (ResultVT.isVector())
    OverflowVT = EVT::getVectorVT(
        *Context, OverflowVT, ResultVT.getVectorElementCount());

  SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
  setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
  return;
}
case Intrinsic::prefetch: {
  SDValue Ops[5];
  unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
  auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
  Ops[0] = DAG.getRoot();
  Ops[1] = getValue(I.getArgOperand(0));
  Ops[2] = getValue(I.getArgOperand(1));
  Ops[3] = getValue(I.getArgOperand(2));
  Ops[4] = getValue(I.getArgOperand(3));
  SDValue Result = DAG.getMemIntrinsicNode(
      ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
      EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
      /* align */ std::nullopt, Flags);

  // Chain the prefetch in parallell with any pending loads, to stay out of
  // the way of later optimizations.
  PendingLoads.push_back(Result);
  Result = getRoot();
  DAG.setRoot(Result);
  return;
}
case Intrinsic::lifetime_start:
case Intrinsic::lifetime_end: {
  bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
  // Stack coloring is not enabled in O0, discard region information.
  if (TM.getOptLevel() == CodeGenOpt::None)
    return;

  const int64_t ObjectSize =
      cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
  Value *const ObjectPtr = I.getArgOperand(1);
  SmallVector<const Value *, 4> Allocas;
  getUnderlyingObjects(ObjectPtr, Allocas);

  for (const Value *Alloca : Allocas) {
    const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);

    // Could not find an Alloca.
    if (!LifetimeObject)
      continue;

    // First check that the Alloca is static, otherwise it won't have a
    // valid frame index.
    auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
    if (SI == FuncInfo.StaticAllocaMap.end())
      return;

    const int FrameIndex = SI->second;
    int64_t Offset;
    if (GetPointerBaseWithConstantOffset(
            ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
      Offset = -1; // Cannot determine offset from alloca to lifetime object.
    Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
                              Offset);
    DAG.setRoot(Res);
  }
  return;
}
case Intrinsic::pseudoprobe: {
  auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
  auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
  auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
  Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
  DAG.setRoot(Res);
  return;
}
case Intrinsic::invariant_start:
  // Discard region information.
  setValue(&I,
           DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
  return;
case Intrinsic::invariant_end:
  // Discard region information.
  return;
case Intrinsic::clear_cache:
  /// FunctionName may be null.
  if (const char *FunctionName = TLI.getClearCacheBuiltinName())
    lowerCallToExternalSymbol(I, FunctionName);
  return;
case Intrinsic::donothing:
case Intrinsic::seh_try_begin:
case Intrinsic::seh_scope_begin:
case Intrinsic::seh_try_end:
case Intrinsic::seh_scope_end:
  // ignore
  return;
case Intrinsic::experimental_stackmap:
  visitStackmap(I);
  return;
case Intrinsic::experimental_patchpoint_void:
case Intrinsic::experimental_patchpoint_i64:
  visitPatchpoint(I);
  return;
case Intrinsic::experimental_gc_statepoint:
  LowerStatepoint(cast<GCStatepointInst>(I));
  return;
case Intrinsic::experimental_gc_result:
  visitGCResult(cast<GCResultInst>(I));
  return;
case Intrinsic::experimental_gc_relocate:
  visitGCRelocate(cast<GCRelocateInst>(I));
  return;
case Intrinsic::instrprof_cover:
  llvm_unreachable("instrprof failed to lower a cover")::llvm::llvm_unreachable_internal("instrprof failed to lower a cover"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7028
);
case Intrinsic::instrprof_increment:
  llvm_unreachable("instrprof failed to lower an increment")::llvm::llvm_unreachable_internal("instrprof failed to lower an increment"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7030
);
case Intrinsic::instrprof_timestamp:
  llvm_unreachable("instrprof failed to lower a timestamp")::llvm::llvm_unreachable_internal("instrprof failed to lower a timestamp"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7032
);
case Intrinsic::instrprof_value_profile:
  llvm_unreachable("instrprof failed to lower a value profiling call")::llvm::llvm_unreachable_internal("instrprof failed to lower a value profiling call"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7034
);
case Intrinsic::localescape: {
  MachineFunction &MF = DAG.getMachineFunction();
  const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();

  // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
  // is the same on all targets.
  for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
    Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
    if (isa<ConstantPointerNull>(Arg))
      continue; // Skip null pointers. They represent a hole in index space.
    AllocaInst *Slot = cast<AllocaInst>(Arg);
    assert(FuncInfo.StaticAllocaMap.count(Slot) &&(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot
) && "can only escape static allocas") ? void (0) : __assert_fail
 ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7047
, __extension__ __PRETTY_FUNCTION__))
           "can only escape static allocas")(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot
) && "can only escape static allocas") ? void (0) : __assert_fail
 ("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7047
, __extension__ __PRETTY_FUNCTION__));
    int FI = FuncInfo.StaticAllocaMap[Slot];
    MCSymbol *FrameAllocSym =
        MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
            GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
            TII->get(TargetOpcode::LOCAL_ESCAPE))
        .addSym(FrameAllocSym)
        .addFrameIndex(FI);
  }

  return;
}

case Intrinsic::localrecover: {
  // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
  MachineFunction &MF = DAG.getMachineFunction();

  // Get the symbol that defines the frame offset.
  auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
  auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
  unsigned IdxVal =
      unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
  MCSymbol *FrameAllocSym =
      MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
          GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);

  Value *FP = I.getArgOperand(1);
  SDValue FPVal = getValue(FP);
  EVT PtrVT = FPVal.getValueType();

  // Create a MCSymbol for the label to avoid any target lowering
  // that would make this PC relative.
  SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
  SDValue OffsetVal =
      DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);

  // Add the offset to the FP.
  SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
  setValue(&I, Add);

  return;
}

case Intrinsic::eh_exceptionpointer:
case Intrinsic::eh_exceptioncode: {
  // Get the exception pointer vreg, copy from it, and resize it to fit.
  const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
  MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
  const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
  unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
  SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
  if (Intrinsic == Intrinsic::eh_exceptioncode)
    N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
  setValue(&I, N);
  return;
}
case Intrinsic::xray_customevent: {
  // Here we want to make sure that the intrinsic behaves as if it has a
  // specific calling convention, and only for x86_64.
  // FIXME: Support other platforms later.
  const auto &Triple = DAG.getTarget().getTargetTriple();
  if (Triple.getArch() != Triple::x86_64)
    return;

  SmallVector<SDValue, 8> Ops;

  // We want to say that we always want the arguments in registers.
  SDValue LogEntryVal = getValue(I.getArgOperand(0));
  SDValue StrSizeVal = getValue(I.getArgOperand(1));
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue Chain = getRoot();
  Ops.push_back(LogEntryVal);
  Ops.push_back(StrSizeVal);
  Ops.push_back(Chain);

  // We need to enforce the calling convention for the callsite, so that
  // argument ordering is enforced correctly, and that register allocation can
  // see that some registers may be assumed clobbered and have to preserve
  // them across calls to the intrinsic.
  MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
                                         sdl, NodeTys, Ops);
  SDValue patchableNode = SDValue(MN, 0);
  DAG.setRoot(patchableNode);
  setValue(&I, patchableNode);
  return;
}
case Intrinsic::xray_typedevent: {
  // Here we want to make sure that the intrinsic behaves as if it has a
  // specific calling convention, and only for x86_64.
  // FIXME: Support other platforms later.
  const auto &Triple = DAG.getTarget().getTargetTriple();
  if (Triple.getArch() != Triple::x86_64)
    return;

  SmallVector<SDValue, 8> Ops;

  // We want to say that we always want the arguments in registers.
  // It's unclear to me how manipulating the selection DAG here forces callers
  // to provide arguments in registers instead of on the stack.
  SDValue LogTypeId = getValue(I.getArgOperand(0));
  SDValue LogEntryVal = getValue(I.getArgOperand(1));
  SDValue StrSizeVal = getValue(I.getArgOperand(2));
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
  SDValue Chain = getRoot();
  Ops.push_back(LogTypeId);
  Ops.push_back(LogEntryVal);
  Ops.push_back(StrSizeVal);
  Ops.push_back(Chain);

  // We need to enforce the calling convention for the callsite, so that
  // argument ordering is enforced correctly, and that register allocation can
  // see that some registers may be assumed clobbered and have to preserve
  // them across calls to the intrinsic.
  MachineSDNode *MN = DAG.getMachineNode(
      TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
  SDValue patchableNode = SDValue(MN, 0);
  DAG.setRoot(patchableNode);
  setValue(&I, patchableNode);
  return;
}
case Intrinsic::experimental_deoptimize:
  LowerDeoptimizeCall(&I);
  return;
case Intrinsic::experimental_stepvector:
  visitStepVector(I);
  return;
case Intrinsic::vector_reduce_fadd:
case Intrinsic::vector_reduce_fmul:
case Intrinsic::vector_reduce_add:
case Intrinsic::vector_reduce_mul:
case Intrinsic::vector_reduce_and:
case Intrinsic::vector_reduce_or:
case Intrinsic::vector_reduce_xor:
case Intrinsic::vector_reduce_smax:
case Intrinsic::vector_reduce_smin:
case Intrinsic::vector_reduce_umax:
case Intrinsic::vector_reduce_umin:
case Intrinsic::vector_reduce_fmax:
case Intrinsic::vector_reduce_fmin:
  visitVectorReduce(I, Intrinsic);
  return;

case Intrinsic::icall_branch_funnel: {
  SmallVector<SDValue, 16> Ops;
  Ops.push_back(getValue(I.getArgOperand(0)));

  int64_t Offset;
  auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
      I.getArgOperand(1), Offset, DAG.getDataLayout()));
  if (!Base)
    report_fatal_error(
        "llvm.icall.branch.funnel operand must be a GlobalValue");
  Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));

  struct BranchFunnelTarget {
    int64_t Offset;
    SDValue Target;
  };
  SmallVector<BranchFunnelTarget, 8> Targets;

  for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
    auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
        I.getArgOperand(Op), Offset, DAG.getDataLayout()));
    if (ElemBase != Base)
      report_fatal_error("all llvm.icall.branch.funnel operands must refer "
                         "to the same GlobalValue");

    SDValue Val = getValue(I.getArgOperand(Op + 1));
    auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
    if (!GA)
      report_fatal_error(
          "llvm.icall.branch.funnel operand must be a GlobalValue");
    Targets.push_back({Offset, DAG.getTargetGlobalAddress(
                                   GA->getGlobal(), sdl, Val.getValueType(),
                                   GA->getOffset())});
  }
  llvm::sort(Targets,
             [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
               return T1.Offset < T2.Offset;
             });

  for (auto &T : Targets) {
    Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
    Ops.push_back(T.Target);
  }

  Ops.push_back(DAG.getRoot()); // Chain
  SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
                               MVT::Other, Ops),
            0);
  DAG.setRoot(N);
  setValue(&I, N);
  HasTailCall = true;
  return;
}

case Intrinsic::wasm_landingpad_index:
  // Information this intrinsic contained has been transferred to
  // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
  // delete it now.
  return;

case Intrinsic::aarch64_settag:
case Intrinsic::aarch64_settag_zero: {
  const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
  bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
  SDValue Val = TSI.EmitTargetCodeForSetTag(
      DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
      getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
      ZeroMemory);
  DAG.setRoot(Val);
  setValue(&I, Val);
  return;
}
case Intrinsic::ptrmask: {
  SDValue Ptr = getValue(I.getOperand(0));
  SDValue Const = getValue(I.getOperand(1));

  EVT PtrVT = Ptr.getValueType();
  setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr,
                           DAG.getZExtOrTrunc(Const, sdl, PtrVT)));
  return;
}
case Intrinsic::threadlocal_address: {
  setValue(&I, getValue(I.getOperand(0)));
  return;
}
case Intrinsic::get_active_lane_mask: {
  EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  SDValue Index = getValue(I.getOperand(0));
  EVT ElementVT = Index.getValueType();

  if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
    visitTargetIntrinsic(I, Intrinsic);
    return;
  }

  SDValue TripCount = getValue(I.getOperand(1));
  auto VecTy = CCVT.changeVectorElementType(ElementVT);

  SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
  SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
  SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
  SDValue VectorInduction = DAG.getNode(
      ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
  SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
                               VectorTripCount, ISD::CondCode::SETULT);
  setValue(&I, SetCC);
  return;
}
case Intrinsic::vector_insert: {
  SDValue Vec = getValue(I.getOperand(0));
  SDValue SubVec = getValue(I.getOperand(1));
  SDValue Index = getValue(I.getOperand(2));

  // The intrinsic's index type is i64, but the SDNode requires an index type
  // suitable for the target. Convert the index as required.
  MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
  if (Index.getValueType() != VectorIdxTy)
    Index = DAG.getVectorIdxConstant(
        cast<ConstantSDNode>(Index)->getZExtValue(), sdl);

  EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
  setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
                           Index));
  return;
}
case Intrinsic::vector_extract: {
  SDValue Vec = getValue(I.getOperand(0));
  SDValue Index = getValue(I.getOperand(1));
  EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

  // The intrinsic's index type is i64, but the SDNode requires an index type
  // suitable for the target. Convert the index as required.
  MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
  if (Index.getValueType() != VectorIdxTy)
    Index = DAG.getVectorIdxConstant(
        cast<ConstantSDNode>(Index)->getZExtValue(), sdl);

  setValue(&I,
           DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
  return;
}
case Intrinsic::experimental_vector_reverse:
  visitVectorReverse(I);
  return;
case Intrinsic::experimental_vector_splice:
  visitVectorSplice(I);
  return;
case Intrinsic::callbr_landingpad:
  visitCallBrLandingPad(I);
  return;
case Intrinsic::experimental_vector_interleave2:
  visitVectorInterleave(I);
  return;
case Intrinsic::experimental_vector_deinterleave2:
  visitVectorDeinterleave(I);
  return;
}
7347}

7349void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
  const ConstrainedFPIntrinsic &FPI) {
SDLoc sdl = getCurSDLoc();

// We do not need to serialize constrained FP intrinsics against
// each other or against (nonvolatile) loads, so they can be
// chained like loads.
SDValue Chain = DAG.getRoot();
SmallVector<SDValue, 4> Opers;
Opers.push_back(Chain);
if (FPI.isUnaryOp()) {
  Opers.push_back(getValue(FPI.getArgOperand(0)));
} else if (FPI.isTernaryOp()) {
  Opers.push_back(getValue(FPI.getArgOperand(0)));
  Opers.push_back(getValue(FPI.getArgOperand(1)));
  Opers.push_back(getValue(FPI.getArgOperand(2)));
} else {
  Opers.push_back(getValue(FPI.getArgOperand(0)));
  Opers.push_back(getValue(FPI.getArgOperand(1)));
}

auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
  assert(Result.getNode()->getNumValues() == 2)(static_cast <bool> (Result.getNode()->getNumValues(
) == 2) ? void (0) : __assert_fail ("Result.getNode()->getNumValues() == 2"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7371
, __extension__ __PRETTY_FUNCTION__));

  // Push node to the appropriate list so that future instructions can be
  // chained up correctly.
  SDValue OutChain = Result.getValue(1);
  switch (EB) {
  case fp::ExceptionBehavior::ebIgnore:
    // The only reason why ebIgnore nodes still need to be chained is that
    // they might depend on the current rounding mode, and therefore must
    // not be moved across instruction that may change that mode.
    [[fallthrough]];
  case fp::ExceptionBehavior::ebMayTrap:
    // These must not be moved across calls or instructions that may change
    // floating-point exception masks.
    PendingConstrainedFP.push_back(OutChain);
    break;
  case fp::ExceptionBehavior::ebStrict:
    // These must not be moved across calls or instructions that may change
    // floating-point exception masks or read floating-point exception flags.
    // In addition, they cannot be optimized out even if unused.
    PendingConstrainedFPStrict.push_back(OutChain);
    break;
  }
};

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
SDVTList VTs = DAG.getVTList(VT, MVT::Other);
fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();

SDNodeFlags Flags;
if (EB == fp::ExceptionBehavior::ebIgnore)
  Flags.setNoFPExcept(true);

if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
  Flags.copyFMF(*FPOp);

unsigned Opcode;
switch (FPI.getIntrinsicID()) {
default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7410);  // Can't reach here.
7411#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
case Intrinsic::INTRINSIC:                                                   \
  Opcode = ISD::STRICT_##DAGN;                                               \
  break;
7415#include "llvm/IR/ConstrainedOps.def"
case Intrinsic::experimental_constrained_fmuladd: {
  Opcode = ISD::STRICT_FMA;
  // Break fmuladd into fmul and fadd.
  if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict ||
      !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
    Opers.pop_back();
    SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
    pushOutChain(Mul, EB);
    Opcode = ISD::STRICT_FADD;
    Opers.clear();
    Opers.push_back(Mul.getValue(1));
    Opers.push_back(Mul.getValue(0));
    Opers.push_back(getValue(FPI.getArgOperand(2)));
  }
  break;
}
}

// A few strict DAG nodes carry additional operands that are not
// set up by the default code above.
switch (Opcode) {
default: break;
case ISD::STRICT_FP_ROUND:
  Opers.push_back(
      DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())));
  break;
case ISD::STRICT_FSETCC:
case ISD::STRICT_FSETCCS: {
  auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
  ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
  if (TM.Options.NoNaNsFPMath)
    Condition = getFCmpCodeWithoutNaN(Condition);
  Opers.push_back(DAG.getCondCode(Condition));
  break;
}
}

SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
pushOutChain(Result, EB);

SDValue FPResult = Result.getValue(0);
setValue(&FPI, FPResult);
7458}

7460static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
std::optional<unsigned> ResOPC;
switch (VPIntrin.getIntrinsicID()) {
case Intrinsic::vp_ctlz: {
  bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
  ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
  break;
}
case Intrinsic::vp_cttz: {
  bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
  ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
  break;
}
7473#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD)                                    \
case Intrinsic::VPID:                                                        \
  ResOPC = ISD::VPSD;                                                        \
  break;
7477#include "llvm/IR/VPIntrinsics.def"
}

if (!ResOPC)
  llvm_unreachable(::llvm::llvm_unreachable_internal("Inconsistency: no SDNode available for this VPIntrinsic!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7482
)
      "Inconsistency: no SDNode available for this VPIntrinsic!")::llvm::llvm_unreachable_internal("Inconsistency: no SDNode available for this VPIntrinsic!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7482
);

if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
    *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
  if (VPIntrin.getFastMathFlags().allowReassoc())
    return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
                                              : ISD::VP_REDUCE_FMUL;
}

return *ResOPC;
7492}

7494void SelectionDAGBuilder::visitVPLoad(
  const VPIntrinsic &VPIntrin, EVT VT,
  const SmallVectorImpl<SDValue> &OpValues) {
SDLoc DL = getCurSDLoc();
Value *PtrOperand = VPIntrin.getArgOperand(0);
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
SDValue LD;
// Do not serialize variable-length loads of constant memory with
// anything.
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT);
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
    MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
                   MMO, false /*IsExpanding */);
if (AddToChain)
  PendingLoads.push_back(LD.getValue(1));
setValue(&VPIntrin, LD);
7518}

7520void SelectionDAGBuilder::visitVPGather(
  const VPIntrinsic &VPIntrin, EVT VT,
  const SmallVectorImpl<SDValue> &OpValues) {
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Value *PtrOperand = VPIntrin.getArgOperand(0);
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
SDValue LD;
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT.getScalarType());
unsigned AS =
  PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
   MachinePointerInfo(AS), MachineMemOperand::MOLoad,
   MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);
SDValue Base, Index, Scale;
ISD::MemIndexType IndexType;
bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
                                  this, VPIntrin.getParent(),
                                  VT.getScalarStoreSize());
if (!UniformBase) {
  Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
  Index = getValue(PtrOperand);
  IndexType = ISD::SIGNED_SCALED;
  Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
}
EVT IdxVT = Index.getValueType();
EVT EltTy = IdxVT.getVectorElementType();
if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
  EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
  Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
}
LD = DAG.getGatherVP(
    DAG.getVTList(VT, MVT::Other), VT, DL,
    {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
    IndexType);
PendingLoads.push_back(LD.getValue(1));
setValue(&VPIntrin, LD);
7560}

7562void SelectionDAGBuilder::visitVPStore(
  const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
SDLoc DL = getCurSDLoc();
Value *PtrOperand = VPIntrin.getArgOperand(1);
EVT VT = OpValues[0].getValueType();
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
SDValue ST;
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT);
SDValue Ptr = OpValues[1];
SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
    MemoryLocation::UnknownSize, *Alignment, AAInfo);
ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
                    OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
                    /* IsTruncating */ false, /*IsCompressing*/ false);
DAG.setRoot(ST);
setValue(&VPIntrin, ST);
7582}

7584void SelectionDAGBuilder::visitVPScatter(
  const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
SDLoc DL = getCurSDLoc();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
Value *PtrOperand = VPIntrin.getArgOperand(1);
EVT VT = OpValues[0].getValueType();
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
SDValue ST;
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT.getScalarType());
unsigned AS =
    PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(AS), MachineMemOperand::MOStore,
    MemoryLocation::UnknownSize, *Alignment, AAInfo);
SDValue Base, Index, Scale;
ISD::MemIndexType IndexType;
bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
                                  this, VPIntrin.getParent(),
                                  VT.getScalarStoreSize());
if (!UniformBase) {
  Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
  Index = getValue(PtrOperand);
  IndexType = ISD::SIGNED_SCALED;
  Scale =
    DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
}
EVT IdxVT = Index.getValueType();
EVT EltTy = IdxVT.getVectorElementType();
if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
  EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
  Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
}
ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
                      {getMemoryRoot(), OpValues[0], Base, Index, Scale,
                       OpValues[2], OpValues[3]},
                      MMO, IndexType);
DAG.setRoot(ST);
setValue(&VPIntrin, ST);
7624}

7626void SelectionDAGBuilder::visitVPStridedLoad(
  const VPIntrinsic &VPIntrin, EVT VT,
  const SmallVectorImpl<SDValue> &OpValues) {
SDLoc DL = getCurSDLoc();
Value *PtrOperand = VPIntrin.getArgOperand(0);
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT.getScalarType());
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range);
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
    MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges);

SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
                                  OpValues[2], OpValues[3], MMO,
                                  false /*IsExpanding*/);

if (AddToChain)
  PendingLoads.push_back(LD.getValue(1));
setValue(&VPIntrin, LD);
7650}

7652void SelectionDAGBuilder::visitVPStridedStore(
  const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
SDLoc DL = getCurSDLoc();
Value *PtrOperand = VPIntrin.getArgOperand(1);
EVT VT = OpValues[0].getValueType();
MaybeAlign Alignment = VPIntrin.getPointerAlignment();
if (!Alignment)
  Alignment = DAG.getEVTAlign(VT.getScalarType());
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
    MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
    MemoryLocation::UnknownSize, *Alignment, AAInfo);

SDValue ST = DAG.getStridedStoreVP(
    getMemoryRoot(), DL, OpValues[0], OpValues[1],
    DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
    OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
    /*IsCompressing*/ false);

DAG.setRoot(ST);
setValue(&VPIntrin, ST);
7673}

7675void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDLoc DL = getCurSDLoc();

ISD::CondCode Condition;
CmpInst::Predicate CondCode = VPIntrin.getPredicate();
bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
if (IsFP) {
  // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
  // flags, but calls that don't return floating-point types can't be
  // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
  Condition = getFCmpCondCode(CondCode);
  if (TM.Options.NoNaNsFPMath)
    Condition = getFCmpCodeWithoutNaN(Condition);
} else {
  Condition = getICmpCondCode(CondCode);
}

SDValue Op1 = getValue(VPIntrin.getOperand(0));
SDValue Op2 = getValue(VPIntrin.getOperand(1));
// #2 is the condition code
SDValue MaskOp = getValue(VPIntrin.getOperand(3));
SDValue EVL = getValue(VPIntrin.getOperand(4));
MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&(static_cast <bool> (EVLParamVT.isScalarInteger() &&
 EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type"
) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7700
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected target EVL type")(static_cast <bool> (EVLParamVT.isScalarInteger() &&
 EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type"
) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7700
, __extension__ __PRETTY_FUNCTION__));
EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);

EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                      VPIntrin.getType());
setValue(&VPIntrin,
         DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
7707}

7709void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
  const VPIntrinsic &VPIntrin) {
SDLoc DL = getCurSDLoc();
unsigned Opcode = getISDForVPIntrinsic(VPIntrin);

auto IID = VPIntrin.getIntrinsicID();

if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
  return visitVPCmp(*CmpI);

SmallVector<EVT, 4> ValueVTs;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
SDVTList VTs = DAG.getVTList(ValueVTs);

auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);

MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&(static_cast <bool> (EVLParamVT.isScalarInteger() &&
 EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type"
) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7728
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected target EVL type")(static_cast <bool> (EVLParamVT.isScalarInteger() &&
 EVLParamVT.bitsGE(MVT::i32) && "Unexpected target EVL type"
) ? void (0) : __assert_fail ("EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && \"Unexpected target EVL type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7728
, __extension__ __PRETTY_FUNCTION__));

// Request operands.
SmallVector<SDValue, 7> OpValues;
for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
  auto Op = getValue(VPIntrin.getArgOperand(I));
  if (I == EVLParamPos)
    Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
  OpValues.push_back(Op);
}

switch (Opcode) {
default: {
  SDNodeFlags SDFlags;
  if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
    SDFlags.copyFMF(*FPMO);
  SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
  setValue(&VPIntrin, Result);
  break;
}
case ISD::VP_LOAD:
  visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
  break;
case ISD::VP_GATHER:
  visitVPGather(VPIntrin, ValueVTs[0], OpValues);
  break;
case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
  visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
  break;
case ISD::VP_STORE:
  visitVPStore(VPIntrin, OpValues);
  break;
case ISD::VP_SCATTER:
  visitVPScatter(VPIntrin, OpValues);
  break;
case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
  visitVPStridedStore(VPIntrin, OpValues);
  break;
case ISD::VP_FMULADD: {
  assert(OpValues.size() == 5 && "Unexpected number of operands")(static_cast <bool> (OpValues.size() == 5 && "Unexpected number of operands"
) ? void (0) : __assert_fail ("OpValues.size() == 5 && \"Unexpected number of operands\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7767
, __extension__ __PRETTY_FUNCTION__));
  SDNodeFlags SDFlags;
  if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
    SDFlags.copyFMF(*FPMO);
  if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
      TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) {
    setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
  } else {
    SDValue Mul = DAG.getNode(
        ISD::VP_FMUL, DL, VTs,
        {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
    SDValue Add =
        DAG.getNode(ISD::VP_FADD, DL, VTs,
                    {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
    setValue(&VPIntrin, Add);
  }
  break;
}
case ISD::VP_INTTOPTR: {
  SDValue N = OpValues[0];
  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
  N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
                             OpValues[2]);
  N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
                           OpValues[2]);
  setValue(&VPIntrin, N);
  break;
}
case ISD::VP_PTRTOINT: {
  SDValue N = OpValues[0];
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                        VPIntrin.getType());
  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
                                     VPIntrin.getOperand(0)->getType());
  N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
                             OpValues[2]);
  N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
                           OpValues[2]);
  setValue(&VPIntrin, N);
  break;
}
case ISD::VP_ABS:
case ISD::VP_CTLZ:
case ISD::VP_CTLZ_ZERO_UNDEF:
case ISD::VP_CTTZ:
case ISD::VP_CTTZ_ZERO_UNDEF: {
  SDValue Result =
      DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
  setValue(&VPIntrin, Result);
  break;
}
}
7820}

7822SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
                                        const BasicBlock *EHPadBB,
                                        MCSymbol *&BeginLabel) {
MachineFunction &MF = DAG.getMachineFunction();
MachineModuleInfo &MMI = MF.getMMI();

// Insert a label before the invoke call to mark the try range.  This can be
// used to detect deletion of the invoke via the MachineModuleInfo.
BeginLabel = MMI.getContext().createTempSymbol();

// For SjLj, keep track of which landing pads go with which invokes
// so as to maintain the ordering of pads in the LSDA.
unsigned CallSiteIndex = MMI.getCurrentCallSite();
if (CallSiteIndex) {
  MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
  LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);

  // Now that the call site is handled, stop tracking it.
  MMI.setCurrentCallSite(0);
}

return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
7844}

7846SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
                                      const BasicBlock *EHPadBB,
                                      MCSymbol *BeginLabel) {
assert(BeginLabel && "BeginLabel should've been set")(static_cast <bool> (BeginLabel && "BeginLabel should've been set"
) ? void (0) : __assert_fail ("BeginLabel && \"BeginLabel should've been set\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7849
, __extension__ __PRETTY_FUNCTION__));

MachineFunction &MF = DAG.getMachineFunction();
MachineModuleInfo &MMI = MF.getMMI();

// Insert a label at the end of the invoke call to mark the try range.  This
// can be used to detect deletion of the invoke via the MachineModuleInfo.
MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);

// Inform MachineModuleInfo of range.
auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
// There is a platform (e.g. wasm) that uses funclet style IR but does not
// actually use outlined funclets and their LSDA info style.
if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
  assert(II && "II should've been set")(static_cast <bool> (II && "II should've been set"
) ? void (0) : __assert_fail ("II && \"II should've been set\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7864
, __extension__ __PRETTY_FUNCTION__));
  WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
  EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
} else if (!isScopedEHPersonality(Pers)) {
  assert(EHPadBB)(static_cast <bool> (EHPadBB) ? void (0) : __assert_fail
 ("EHPadBB", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7868, __extension__ __PRETTY_FUNCTION__));
  MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
}

return Chain;
7873}

7875std::pair<SDValue, SDValue>
7876SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
                                  const BasicBlock *EHPadBB) {
MCSymbol *BeginLabel = nullptr;

if (EHPadBB) {
  // Both PendingLoads and PendingExports must be flushed here;
  // this call might not return.
  (void)getRoot();
  DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
  CLI.setChain(getRoot());
}

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);

assert((CLI.IsTailCall || Result.second.getNode()) &&(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode
()) && "Non-null chain expected with non-tail call!")
 ? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7892
, __extension__ __PRETTY_FUNCTION__))
       "Non-null chain expected with non-tail call!")(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode
()) && "Non-null chain expected with non-tail call!")
 ? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7892
, __extension__ __PRETTY_FUNCTION__));
assert((Result.second.getNode() || !Result.first.getNode()) &&(static_cast <bool> ((Result.second.getNode() || !Result
.first.getNode()) && "Null value expected with tail call!"
) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7894
, __extension__ __PRETTY_FUNCTION__))
       "Null value expected with tail call!")(static_cast <bool> ((Result.second.getNode() || !Result
.first.getNode()) && "Null value expected with tail call!"
) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 7894
, __extension__ __PRETTY_FUNCTION__));

if (!Result.second.getNode()) {
  // As a special case, a null chain means that a tail call has been emitted
  // and the DAG root is already updated.
  HasTailCall = true;

  // Since there's no actual continuation from this block, nothing can be
  // relying on us setting vregs for them.
  PendingExports.clear();
} else {
  DAG.setRoot(Result.second);
}

if (EHPadBB) {
  DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
                         BeginLabel));
}

return Result;
7914}

7916void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
                                    bool isTailCall,
                                    bool isMustTailCall,
                                    const BasicBlock *EHPadBB) {
auto &DL = DAG.getDataLayout();
FunctionType *FTy = CB.getFunctionType();
Type *RetTy = CB.getType();

TargetLowering::ArgListTy Args;
Args.reserve(CB.arg_size());

const Value *SwiftErrorVal = nullptr;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

if (isTailCall) {
  // Avoid emitting tail calls in functions with the disable-tail-calls
  // attribute.
  auto *Caller = CB.getParent()->getParent();
  if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
      "true" && !isMustTailCall)
    isTailCall = false;

  // We can't tail call inside a function with a swifterror argument. Lowering
  // does not support this yet. It would have to move into the swifterror
  // register before the call.
  if (TLI.supportSwiftError() &&
      Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
    isTailCall = false;
}

for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
  TargetLowering::ArgListEntry Entry;
  const Value *V = *I;

  // Skip empty types
  if (V->getType()->isEmptyTy())
    continue;

  SDValue ArgNode = getValue(V);
  Entry.Node = ArgNode; Entry.Ty = V->getType();

  Entry.setAttributes(&CB, I - CB.arg_begin());

  // Use swifterror virtual register as input to the call.
  if (Entry.IsSwiftError && TLI.supportSwiftError()) {
    SwiftErrorVal = V;
    // We find the virtual register for the actual swifterror argument.
    // Instead of using the Value, we use the virtual register instead.
    Entry.Node =
        DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V),
                        EVT(TLI.getPointerTy(DL)));
  }

  Args.push_back(Entry);

  // If we have an explicit sret argument that is an Instruction, (i.e., it
  // might point to function-local memory), we can't meaningfully tail-call.
  if (Entry.IsSRet && isa<Instruction>(V))
    isTailCall = false;
}

// If call site has a cfguardtarget operand bundle, create and add an
// additional ArgListEntry.
if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
  TargetLowering::ArgListEntry Entry;
  Value *V = Bundle->Inputs[0];
  SDValue ArgNode = getValue(V);
  Entry.Node = ArgNode;
  Entry.Ty = V->getType();
  Entry.IsCFGuardTarget = true;
  Args.push_back(Entry);
}

// Check if target-independent constraints permit a tail call here.
// Target-dependent constraints are checked within TLI->LowerCallTo.
if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
  isTailCall = false;

// Disable tail calls if there is an swifterror argument. Targets have not
// been updated to support tail calls.
if (TLI.supportSwiftError() && SwiftErrorVal)
  isTailCall = false;

ConstantInt *CFIType = nullptr;
if (CB.isIndirectCall()) {
  if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
    if (!TLI.supportKCFIBundles())
      report_fatal_error(
          "Target doesn't support calls with kcfi operand bundles.");
    CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
    assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type")(static_cast <bool> (CFIType->getType()->isIntegerTy
(32) && "Invalid CFI type") ? void (0) : __assert_fail
 ("CFIType->getType()->isIntegerTy(32) && \"Invalid CFI type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8006
, __extension__ __PRETTY_FUNCTION__));
  }
}

TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(getCurSDLoc())
    .setChain(getRoot())
    .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
    .setTailCall(isTailCall)
    .setConvergent(CB.isConvergent())
    .setIsPreallocated(
        CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
    .setCFIType(CFIType);
std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);

if (Result.first.getNode()) {
  Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
  setValue(&CB, Result.first);
}

// The last element of CLI.InVals has the SDValue for swifterror return.
// Here we copy it to a virtual register and update SwiftErrorMap for
// book-keeping.
if (SwiftErrorVal && TLI.supportSwiftError()) {
  // Get the last element of InVals.
  SDValue Src = CLI.InVals.back();
  Register VReg =
      SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
  SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
  DAG.setRoot(CopyNode);
}
8037}

8039static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
                           SelectionDAGBuilder &Builder) {
// Check to see if this load can be trivially constant folded, e.g. if the
// input is from a string literal.
if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
  // Cast pointer to the type we really want to load.
  Type *LoadTy =
      Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
  if (LoadVT.isVector())
    LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());

  LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
                                       PointerType::getUnqual(LoadTy));

  if (const Constant *LoadCst =
          ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
                                       LoadTy, Builder.DAG.getDataLayout()))
    return Builder.getValue(LoadCst);
}

// Otherwise, we have to emit the load.  If the pointer is to unfoldable but
// still constant memory, the input chain can be the entry node.
SDValue Root;
bool ConstantMemory = false;

// Do not serialize (non-volatile) loads of constant memory with anything.
if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
  Root = Builder.DAG.getEntryNode();
  ConstantMemory = true;
} else {
  // Do not serialize non-volatile loads against each other.
  Root = Builder.DAG.getRoot();
}

SDValue Ptr = Builder.getValue(PtrVal);
SDValue LoadVal =
    Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
                        MachinePointerInfo(PtrVal), Align(1));

if (!ConstantMemory)
  Builder.PendingLoads.push_back(LoadVal.getValue(1));
return LoadVal;
8081}

8083/// Record the value for an instruction that produces an integer result,
8084/// converting the type where necessary.
8085void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
                                                SDValue Value,
                                                bool IsSigned) {
EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                  I.getType(), true);
if (IsSigned)
  Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
else
  Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
setValue(&I, Value);
8095}

8097/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
8098/// true and lower it. Otherwise return false, and it will be lowered like a
8099/// normal call.
8100/// The caller already checked that \p I calls the appropriate LibFunc with a
8101/// correct prototype.
8102bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
const Value *Size = I.getArgOperand(2);
const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
if (CSize && CSize->getZExtValue() == 0) {
  EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
                                                        I.getType(), true);
  setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
  return true;
}

const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
    DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
    getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
if (Res.first.getNode()) {
  processIntegerCallValue(I, Res.first, true);
  PendingLoads.push_back(Res.second);
  return true;
}

// memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS)  != 0
// memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS)  != 0
if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
  return false;

// If the target has a fast compare for the given size, it will return a
// preferred load type for that size. Require that the load VT is legal and
// that the target supports unaligned loads of that type. Otherwise, return
// INVALID.
auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  MVT LVT = TLI.hasFastEqualityCompare(NumBits);
  if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
    // TODO: Handle 5 byte compare as 4-byte + 1 byte.
    // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
    // TODO: Check alignment of src and dest ptrs.
    unsigned DstAS = LHS->getType()->getPointerAddressSpace();
    unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
    if (!TLI.isTypeLegal(LVT) ||
        !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
        !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
      LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
  }

  return LVT;
};

// This turns into unaligned loads. We only do this if the target natively
// supports the MVT we'll be loading or if it is small enough (<= 4) that
// we'll only produce a small number of byte loads.
MVT LoadVT;
unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
switch (NumBitsToCompare) {
default:
  return false;
case 16:
  LoadVT = MVT::i16;
  break;
case 32:
  LoadVT = MVT::i32;
  break;
case 64:
case 128:
case 256:
  LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
  break;
}

if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
  return false;

SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);

// Bitcast to a wide integer type if the loads are vectors.
if (LoadVT.isVector()) {
  EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
  LoadL = DAG.getBitcast(CmpVT, LoadL);
  LoadR = DAG.getBitcast(CmpVT, LoadR);
}

SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
processIntegerCallValue(I, Cmp, false);
return true;
8187}

8189/// See if we can lower a memchr call into an optimized form. If so, return
8190/// true and lower it. Otherwise return false, and it will be lowered like a
8191/// normal call.
8192/// The caller already checked that \p I calls the appropriate LibFunc with a
8193/// correct prototype.
8194bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
const Value *Src = I.getArgOperand(0);
const Value *Char = I.getArgOperand(1);
const Value *Length = I.getArgOperand(2);

const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
std::pair<SDValue, SDValue> Res =
  TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
                              getValue(Src), getValue(Char), getValue(Length),
                              MachinePointerInfo(Src));
if (Res.first.getNode()) {
  setValue(&I, Res.first);
  PendingLoads.push_back(Res.second);
  return true;
}

return false;
8211}

8213/// See if we can lower a mempcpy call into an optimized form. If so, return
8214/// true and lower it. Otherwise return false, and it will be lowered like a
8215/// normal call.
8216/// The caller already checked that \p I calls the appropriate LibFunc with a
8217/// correct prototype.
8218bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
SDValue Dst = getValue(I.getArgOperand(0));
SDValue Src = getValue(I.getArgOperand(1));
SDValue Size = getValue(I.getArgOperand(2));

Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
// DAG::getMemcpy needs Alignment to be defined.
Align Alignment = std::min(DstAlign, SrcAlign);

bool isVol = false;
SDLoc sdl = getCurSDLoc();

// In the mempcpy context we need to pass in a false value for isTailCall
// because the return pointer needs to be adjusted by the size of
// the copied memory.
SDValue Root = isVol ? getRoot() : getMemoryRoot();
SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false,
                           /*isTailCall=*/false,
                           MachinePointerInfo(I.getArgOperand(0)),
                           MachinePointerInfo(I.getArgOperand(1)),
                           I.getAAMetadata());
assert(MC.getNode() != nullptr &&(static_cast <bool> (MC.getNode() != nullptr &&
 "** memcpy should not be lowered as TailCall in mempcpy context **"
) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8241
, __extension__ __PRETTY_FUNCTION__))
       "** memcpy should not be lowered as TailCall in mempcpy context **")(static_cast <bool> (MC.getNode() != nullptr &&
 "** memcpy should not be lowered as TailCall in mempcpy context **"
) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8241
, __extension__ __PRETTY_FUNCTION__));
DAG.setRoot(MC);

// Check if Size needs to be truncated or extended.
Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());

// Adjust return pointer to point just past the last dst byte.
SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
                                  Dst, Size);
setValue(&I, DstPlusSize);
return true;
8252}

8254/// See if we can lower a strcpy call into an optimized form.  If so, return
8255/// true and lower it, otherwise return false and it will be lowered like a
8256/// normal call.
8257/// The caller already checked that \p I calls the appropriate LibFunc with a
8258/// correct prototype.
8259bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);

const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
std::pair<SDValue, SDValue> Res =
  TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
                              getValue(Arg0), getValue(Arg1),
                              MachinePointerInfo(Arg0),
                              MachinePointerInfo(Arg1), isStpcpy);
if (Res.first.getNode()) {
  setValue(&I, Res.first);
  DAG.setRoot(Res.second);
  return true;
}

return false;
8275}

8277/// See if we can lower a strcmp call into an optimized form.  If so, return
8278/// true and lower it, otherwise return false and it will be lowered like a
8279/// normal call.
8280/// The caller already checked that \p I calls the appropriate LibFunc with a
8281/// correct prototype.
8282bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);

const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
std::pair<SDValue, SDValue> Res =
  TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
                              getValue(Arg0), getValue(Arg1),
                              MachinePointerInfo(Arg0),
                              MachinePointerInfo(Arg1));
if (Res.first.getNode()) {
  processIntegerCallValue(I, Res.first, true);
  PendingLoads.push_back(Res.second);
  return true;
}

return false;
8298}

8300/// See if we can lower a strlen call into an optimized form.  If so, return
8301/// true and lower it, otherwise return false and it will be lowered like a
8302/// normal call.
8303/// The caller already checked that \p I calls the appropriate LibFunc with a
8304/// correct prototype.
8305bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0);

const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
std::pair<SDValue, SDValue> Res =
  TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
                              getValue(Arg0), MachinePointerInfo(Arg0));
if (Res.first.getNode()) {
  processIntegerCallValue(I, Res.first, false);
  PendingLoads.push_back(Res.second);
  return true;
}

return false;
8319}

8321/// See if we can lower a strnlen call into an optimized form.  If so, return
8322/// true and lower it, otherwise return false and it will be lowered like a
8323/// normal call.
8324/// The caller already checked that \p I calls the appropriate LibFunc with a
8325/// correct prototype.
8326bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);

const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
std::pair<SDValue, SDValue> Res =
  TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
                               getValue(Arg0), getValue(Arg1),
                               MachinePointerInfo(Arg0));
if (Res.first.getNode()) {
  processIntegerCallValue(I, Res.first, false);
  PendingLoads.push_back(Res.second);
  return true;
}

return false;
8341}

8343/// See if we can lower a unary floating-point operation into an SDNode with
8344/// the specified Opcode.  If so, return true and lower it, otherwise return
8345/// false and it will be lowered like a normal call.
8346/// The caller already checked that \p I calls the appropriate LibFunc with a
8347/// correct prototype.
8348bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
                                            unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
  return false;

SDNodeFlags Flags;
Flags.copyFMF(cast<FPMathOperator>(I));

SDValue Tmp = getValue(I.getArgOperand(0));
setValue(&I,
         DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
return true;
8361}

8363/// See if we can lower a binary floating-point operation into an SDNode with
8364/// the specified Opcode. If so, return true and lower it. Otherwise return
8365/// false, and it will be lowered like a normal call.
8366/// The caller already checked that \p I calls the appropriate LibFunc with a
8367/// correct prototype.
8368bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
                                             unsigned Opcode) {
// We already checked this call's prototype; verify it doesn't modify errno.
if (!I.onlyReadsMemory())
  return false;

SDNodeFlags Flags;
Flags.copyFMF(cast<FPMathOperator>(I));

SDValue Tmp0 = getValue(I.getArgOperand(0));
SDValue Tmp1 = getValue(I.getArgOperand(1));
EVT VT = Tmp0.getValueType();
setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
return true;
8382}

8384void SelectionDAGBuilder::visitCall(const CallInst &I) {
// Handle inline assembly differently.
if (I.isInlineAsm()) {
  visitInlineAsm(I);
  return;
}

diagnoseDontCall(I);

if (Function *F = I.getCalledFunction()) {
  if (F->isDeclaration()) {
    // Is this an LLVM intrinsic or a target-specific intrinsic?
    unsigned IID = F->getIntrinsicID();
    if (!IID)
      if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
        IID = II->getIntrinsicID(F);

    if (IID) {
      visitIntrinsicCall(I, IID);
      return;
    }
  }

  // Check for well-known libc/libm calls.  If the function is internal, it
  // can't be a library call.  Don't do the check if marked as nobuiltin for
  // some reason or the call site requires strict floating point semantics.
  LibFunc Func;
  if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
      F->hasName() && LibInfo->getLibFunc(*F, Func) &&
      LibInfo->hasOptimizedCodeGen(Func)) {
    switch (Func) {
    default: break;
    case LibFunc_bcmp:
      if (visitMemCmpBCmpCall(I))
        return;
      break;
    case LibFunc_copysign:
    case LibFunc_copysignf:
    case LibFunc_copysignl:
      // We already checked this call's prototype; verify it doesn't modify
      // errno.
      if (I.onlyReadsMemory()) {
        SDValue LHS = getValue(I.getArgOperand(0));
        SDValue RHS = getValue(I.getArgOperand(1));
        setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
                                 LHS.getValueType(), LHS, RHS));
        return;
      }
      break;
    case LibFunc_fabs:
    case LibFunc_fabsf:
    case LibFunc_fabsl:
      if (visitUnaryFloatCall(I, ISD::FABS))
        return;
      break;
    case LibFunc_fmin:
    case LibFunc_fminf:
    case LibFunc_fminl:
      if (visitBinaryFloatCall(I, ISD::FMINNUM))
        return;
      break;
    case LibFunc_fmax:
    case LibFunc_fmaxf:
    case LibFunc_fmaxl:
      if (visitBinaryFloatCall(I, ISD::FMAXNUM))
        return;
      break;
    case LibFunc_sin:
    case LibFunc_sinf:
    case LibFunc_sinl:
      if (visitUnaryFloatCall(I, ISD::FSIN))
        return;
      break;
    case LibFunc_cos:
    case LibFunc_cosf:
    case LibFunc_cosl:
      if (visitUnaryFloatCall(I, ISD::FCOS))
        return;
      break;
    case LibFunc_sqrt:
    case LibFunc_sqrtf:
    case LibFunc_sqrtl:
    case LibFunc_sqrt_finite:
    case LibFunc_sqrtf_finite:
    case LibFunc_sqrtl_finite:
      if (visitUnaryFloatCall(I, ISD::FSQRT))
        return;
      break;
    case LibFunc_floor:
    case LibFunc_floorf:
    case LibFunc_floorl:
      if (visitUnaryFloatCall(I, ISD::FFLOOR))
        return;
      break;
    case LibFunc_nearbyint:
    case LibFunc_nearbyintf:
    case LibFunc_nearbyintl:
      if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
        return;
      break;
    case LibFunc_ceil:
    case LibFunc_ceilf:
    case LibFunc_ceill:
      if (visitUnaryFloatCall(I, ISD::FCEIL))
        return;
      break;
    case LibFunc_rint:
    case LibFunc_rintf:
    case LibFunc_rintl:
      if (visitUnaryFloatCall(I, ISD::FRINT))
        return;
      break;
    case LibFunc_round:
    case LibFunc_roundf:
    case LibFunc_roundl:
      if (visitUnaryFloatCall(I, ISD::FROUND))
        return;
      break;
    case LibFunc_trunc:
    case LibFunc_truncf:
    case LibFunc_truncl:
      if (visitUnaryFloatCall(I, ISD::FTRUNC))
        return;
      break;
    case LibFunc_log2:
    case LibFunc_log2f:
    case LibFunc_log2l:
      if (visitUnaryFloatCall(I, ISD::FLOG2))
        return;
      break;
    case LibFunc_exp2:
    case LibFunc_exp2f:
    case LibFunc_exp2l:
      if (visitUnaryFloatCall(I, ISD::FEXP2))
        return;
      break;
    case LibFunc_memcmp:
      if (visitMemCmpBCmpCall(I))
        return;
      break;
    case LibFunc_mempcpy:
      if (visitMemPCpyCall(I))
        return;
      break;
    case LibFunc_memchr:
      if (visitMemChrCall(I))
        return;
      break;
    case LibFunc_strcpy:
      if (visitStrCpyCall(I, false))
        return;
      break;
    case LibFunc_stpcpy:
      if (visitStrCpyCall(I, true))
        return;
      break;
    case LibFunc_strcmp:
      if (visitStrCmpCall(I))
        return;
      break;
    case LibFunc_strlen:
      if (visitStrLenCall(I))
        return;
      break;
    case LibFunc_strnlen:
      if (visitStrNLenCall(I))
        return;
      break;
    }
  }
}

// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
// CFGuardTarget bundles are lowered in LowerCallTo.
assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget
, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall
, LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563
, __extension__ __PRETTY_FUNCTION__))
           {LLVMContext::OB_deopt, LLVMContext::OB_funclet,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget
, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall
, LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563
, __extension__ __PRETTY_FUNCTION__))
            LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget
, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall
, LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563
, __extension__ __PRETTY_FUNCTION__))
            LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget
, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall
, LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563
, __extension__ __PRETTY_FUNCTION__))
       "Cannot lower calls with arbitrary operand bundles!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget
, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall
, LLVMContext::OB_kcfi}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8563
, __extension__ __PRETTY_FUNCTION__));

SDValue Callee = getValue(I.getCalledOperand());

if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
  LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
else
  // Check if we can potentially perform a tail call. More detailed checking
  // is be done within LowerCallTo, after more information about the call is
  // known.
  LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
8574}

8576namespace {

8578/// AsmOperandInfo - This contains information for each constraint that we are
8579/// lowering.
8580class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
8581public:
/// CallOperand - If this is the result output operand or a clobber
/// this is null, otherwise it is the incoming operand to the CallInst.
/// This gets modified as the asm is processed.
SDValue CallOperand;

/// AssignedRegs - If this is a register or register class operand, this
/// contains the set of register corresponding to the operand.
RegsForValue AssignedRegs;

explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
  : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
}

/// Whether or not this operand accesses memory
bool hasMemory(const TargetLowering &TLI) const {
  // Indirect operand accesses access memory.
  if (isIndirect)
    return true;

  for (const auto &Code : Codes)
    if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
      return true;

  return false;
}
8607};


8610} // end anonymous namespace

8612/// Make sure that the output operand \p OpInfo and its corresponding input
8613/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
8614/// out).
8615static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
                             SDISelAsmOperandInfo &MatchingOpInfo,
                             SelectionDAG &DAG) {
if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
  return;

const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
const auto &TLI = DAG.getTargetLoweringInfo();

std::pair<unsigned, const TargetRegisterClass *> MatchRC =
    TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
                                     OpInfo.ConstraintVT);
std::pair<unsigned, const TargetRegisterClass *> InputRC =
    TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
                                     MatchingOpInfo.ConstraintVT);
if ((OpInfo.ConstraintVT.isInteger() !=
     MatchingOpInfo.ConstraintVT.isInteger()) ||
    (MatchRC.second != InputRC.second)) {
  // FIXME: error out in a more elegant fashion
  report_fatal_error("Unsupported asm: input constraint"
                     " with a matching output constraint of"
                     " incompatible type!");
}
MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
8639}

8641/// Get a direct memory input to behave well as an indirect operand.
8642/// This may introduce stores, hence the need for a \p Chain.
8643/// \return The (possibly updated) chain.
8644static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
                                      SDISelAsmOperandInfo &OpInfo,
                                      SelectionDAG &DAG) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

// If we don't have an indirect input, put it in the constpool if we can,
// otherwise spill it to a stack slot.
// TODO: This isn't quite right. We need to handle these according to
// the addressing mode that the constraint wants. Also, this may take
// an additional register for the computation and we don't want that
// either.

// If the operand is a float, integer, or vector constant, spill to a
// constant pool entry to get its address.
const Value *OpVal = OpInfo.CallOperandVal;
if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
    isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
  OpInfo.CallOperand = DAG.getConstantPool(
      cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
  return Chain;
}

// Otherwise, create a stack slot and emit a store to it before the asm.
Type *Ty = OpVal->getType();
auto &DL = DAG.getDataLayout();
uint64_t TySize = DL.getTypeAllocSize(Ty);
MachineFunction &MF = DAG.getMachineFunction();
int SSFI = MF.getFrameInfo().CreateStackObject(
    TySize, DL.getPrefTypeAlign(Ty), false);
SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
                          MachinePointerInfo::getFixedStack(MF, SSFI),
                          TLI.getMemValueType(DL, Ty));
OpInfo.CallOperand = StackSlot;

return Chain;
8680}

8682/// GetRegistersForValue - Assign registers (virtual or physical) for the
8683/// specified operand.  We prefer to assign virtual registers, to allow the
8684/// register allocator to handle the assignment process.  However, if the asm
8685/// uses features that we can't model on machineinstrs, we have SDISel do the
8686/// allocation.  This produces generally horrible, but correct, code.
8687///
8688///   OpInfo describes the operand
8689///   RefOpInfo describes the matching operand if any, the operand otherwise
8690static std::optional<unsigned>
8691getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
                   SDISelAsmOperandInfo &OpInfo,
                   SDISelAsmOperandInfo &RefOpInfo) {
LLVMContext &Context = *DAG.getContext();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

MachineFunction &MF = DAG.getMachineFunction();
SmallVector<unsigned, 4> Regs;
const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();

// No work to do for memory/address operands.
if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
    OpInfo.ConstraintType == TargetLowering::C_Address)
  return std::nullopt;

// If this is a constraint for a single physreg, or a constraint for a
// register class, find it.
unsigned AssignedReg;
const TargetRegisterClass *RC;
std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
    &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
// RC is unset only on failure. Return immediately.
if (!RC)
  return std::nullopt;

// Get the actual register value type.  This is important, because the user
// may have asked for (e.g.) the AX register in i32 type.  We need to
// remember that AX is actually i16 to get the right extension.
const MVT RegVT = *TRI.legalclasstypes_begin(*RC);

if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
  // If this is an FP operand in an integer register (or visa versa), or more
  // generally if the operand value disagrees with the register class we plan
  // to stick it in, fix the operand type.
  //
  // If this is an input value, the bitcast to the new type is done now.
  // Bitcast for output value is done at the end of visitInlineAsm().
  if ((OpInfo.Type == InlineAsm::isOutput ||
       OpInfo.Type == InlineAsm::isInput) &&
      !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
    // Try to convert to the first EVT that the reg class contains.  If the
    // types are identical size, use a bitcast to convert (e.g. two differing
    // vector types).  Note: output bitcast is done at the end of
    // visitInlineAsm().
    if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
      // Exclude indirect inputs while they are unsupported because the code
      // to perform the load is missing and thus OpInfo.CallOperand still
      // refers to the input address rather than the pointed-to value.
      if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
        OpInfo.CallOperand =
            DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
      OpInfo.ConstraintVT = RegVT;
      // If the operand is an FP value and we want it in integer registers,
      // use the corresponding integer type. This turns an f64 value into
      // i64, which can be passed with two i32 values on a 32-bit machine.
    } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
      MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
      if (OpInfo.Type == InlineAsm::isInput)
        OpInfo.CallOperand =
            DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
      OpInfo.ConstraintVT = VT;
    }
  }
}

// No need to allocate a matching input constraint since the constraint it's
// matching to has already been allocated.
if (OpInfo.isMatchingInputConstraint())
  return std::nullopt;

EVT ValueVT = OpInfo.ConstraintVT;
if (OpInfo.ConstraintVT == MVT::Other)
  ValueVT = RegVT;

// Initialize NumRegs.
unsigned NumRegs = 1;
if (OpInfo.ConstraintVT != MVT::Other)
  NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);

// If this is a constraint for a specific physical register, like {r17},
// assign it now.

// If this associated to a specific register, initialize iterator to correct
// place. If virtual, make sure we have enough registers

// Initialize iterator if necessary
TargetRegisterClass::iterator I = RC->begin();
MachineRegisterInfo &RegInfo = MF.getRegInfo();

// Do not check for single registers.
if (AssignedReg) {
  I = std::find(I, RC->end(), AssignedReg);
  if (I == RC->end()) {
    // RC does not contain the selected register, which indicates a
    // mismatch between the register and the required type/bitwidth.
    return {AssignedReg};
  }
}

for (; NumRegs; --NumRegs, ++I) {
  assert(I != RC->end() && "Ran out of registers to allocate!")(static_cast <bool> (I != RC->end() && "Ran out of registers to allocate!"
) ? void (0) : __assert_fail ("I != RC->end() && \"Ran out of registers to allocate!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8791
, __extension__ __PRETTY_FUNCTION__));
  Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
  Regs.push_back(R);
}

OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
return std::nullopt;
8798}

8800static unsigned
8801findMatchingInlineAsmOperand(unsigned OperandNo,
                           const std::vector<SDValue> &AsmNodeOperands) {
// Scan until we find the definition we already emitted of this operand.
unsigned CurOp = InlineAsm::Op_FirstOperand;
for (; OperandNo; --OperandNo) {
  // Advance to the next operand.
  unsigned OpFlag =
      cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
  assert((InlineAsm::isRegDefKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
 __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812
, __extension__ __PRETTY_FUNCTION__))
          InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
 __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812
, __extension__ __PRETTY_FUNCTION__))
          InlineAsm::isMemKind(OpFlag)) &&(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
 __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812
, __extension__ __PRETTY_FUNCTION__))
         "Skipped past definitions?")(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
 __assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8812
, __extension__ __PRETTY_FUNCTION__));
  CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
}
return CurOp;
8816}

8818namespace {

8820class ExtraFlags {
unsigned Flags = 0;

8823public:
explicit ExtraFlags(const CallBase &Call) {
  const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
  if (IA->hasSideEffects())
    Flags |= InlineAsm::Extra_HasSideEffects;
  if (IA->isAlignStack())
    Flags |= InlineAsm::Extra_IsAlignStack;
  if (Call.isConvergent())
    Flags |= InlineAsm::Extra_IsConvergent;
  Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
}

void update(const TargetLowering::AsmOperandInfo &OpInfo) {
  // Ideally, we would only check against memory constraints.  However, the
  // meaning of an Other constraint can be target-specific and we can't easily
  // reason about it.  Therefore, be conservative and set MayLoad/MayStore
  // for Other constraints as well.
  if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
      OpInfo.ConstraintType == TargetLowering::C_Other) {
    if (OpInfo.Type == InlineAsm::isInput)
      Flags |= InlineAsm::Extra_MayLoad;
    else if (OpInfo.Type == InlineAsm::isOutput)
      Flags |= InlineAsm::Extra_MayStore;
    else if (OpInfo.Type == InlineAsm::isClobber)
      Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
  }
}

unsigned get() const { return Flags; }
8852};

8854} // end anonymous namespace

8856static bool isFunction(SDValue Op) {
if (Op && Op.getOpcode() == ISD::GlobalAddress) {
  if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
    auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());

    // In normal "call dllimport func" instruction (non-inlineasm) it force
    // indirect access by specifing call opcode. And usually specially print
    // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
    // not do in this way now. (In fact, this is similar with "Data Access"
    // action). So here we ignore dllimport function.
    if (Fn && !Fn->hasDLLImportStorageClass())
      return true;
  }
}
return false;
8871}

8873/// visitInlineAsm - Handle a call to an InlineAsm object.
8874void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
                                       const BasicBlock *EHPadBB) {
const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
1
The object is a 'CastReturnType'→

/// ConstraintOperands - Information about all of the constraints.
SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
    DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call);

// First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
// AsmDialect, MayLoad, MayStore).
bool HasSideEffect = IA->hasSideEffects();
ExtraFlags ExtraInfo(Call);

for (auto &T : TargetConstraints) {
  ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
  SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();

  if (OpInfo.CallOperandVal)
    OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);

  if (!HasSideEffect)
    HasSideEffect = OpInfo.hasMemory(TLI);

  // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
  // FIXME: Could we compute this on OpInfo rather than T?

  // Compute the constraint code and ConstraintType to use.
  TLI.ComputeConstraintToUse(T, SDValue());

  if (T.ConstraintType == TargetLowering::C_Immediate &&
      OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
    // We've delayed emitting a diagnostic like the "n" constraint because
    // inlining could cause an integer showing up.
    return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
                                        "' expects an integer constant "
                                        "expression");

  ExtraInfo.update(T);
}

// We won't need to flush pending loads if this asm doesn't touch
// memory and is nonvolatile.
SDValue Glue, Chain = (HasSideEffect1.1
'HasSideEffect' is false
1
'HasSideEffect' is false
1
'HasSideEffect' is false
) ? getRoot() : DAG.getRoot();
2
←
'?' condition is false→

bool EmitEHLabels = isa<InvokeInst>(Call);
3
←
Assuming 'Call' is not a 'InvokeInst'→
if (EmitEHLabels3.1
'EmitEHLabels' is false
1
'EmitEHLabels' is false
1
'EmitEHLabels' is false
) {
4
←
Taking false branch→
  assert(EHPadBB && "InvokeInst must have an EHPadBB")(static_cast <bool> (EHPadBB && "InvokeInst must have an EHPadBB"
) ? void (0) : __assert_fail ("EHPadBB && \"InvokeInst must have an EHPadBB\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8923
, __extension__ __PRETTY_FUNCTION__));
}
bool IsCallBr = isa<CallBrInst>(Call);
5
←
Assuming 'Call' is not a 'CallBrInst'→

if (IsCallBr5.1
'IsCallBr' is false
1
'IsCallBr' is false
1
'IsCallBr' is false
 || EmitEHLabels5.2
'EmitEHLabels' is false
2
'EmitEHLabels' is false
2
'EmitEHLabels' is false
) {
6
←
Taking false branch→
  // If this is a callbr or invoke we need to flush pending exports since
  // inlineasm_br and invoke are terminators.
  // We need to do this before nodes are glued to the inlineasm_br node.
  Chain = getControlRoot();
}

MCSymbol *BeginLabel = nullptr;
if (EmitEHLabels6.1
'EmitEHLabels' is false
1
'EmitEHLabels' is false
1
'EmitEHLabels' is false
) {
7
←
Taking false branch→
  Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
}

int OpNo = -1;
SmallVector<StringRef> AsmStrs;
IA->collectAsmStrs(AsmStrs);

// Second pass over the constraints: compute which constraint option to use.
for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
8
←
Assuming '__begin1' is equal to '__end1'→
  if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
    OpNo++;

  // If this is an output operand with a matching input operand, look up the
  // matching input. If their types mismatch, e.g. one is an integer, the
  // other is floating point, or their sizes are different, flag it as an
  // error.
  if (OpInfo.hasMatchingInput()) {
    SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
    patchMatchingInput(OpInfo, Input, DAG);
  }

  // Compute the constraint code and ConstraintType to use.
  TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);

  if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
       OpInfo.Type == InlineAsm::isClobber) ||
      OpInfo.ConstraintType == TargetLowering::C_Address)
    continue;

  // In Linux PIC model, there are 4 cases about value/label addressing:
  //
  // 1: Function call or Label jmp inside the module.
  // 2: Data access (such as global variable, static variable) inside module.
  // 3: Function call or Label jmp outside the module.
  // 4: Data access (such as global variable) outside the module.
  //
  // Due to current llvm inline asm architecture designed to not "recognize"
  // the asm code, there are quite troubles for us to treat mem addressing
  // differently for same value/adress used in different instuctions.
  // For example, in pic model, call a func may in plt way or direclty
  // pc-related, but lea/mov a function adress may use got.
  //
  // Here we try to "recognize" function call for the case 1 and case 3 in
  // inline asm. And try to adjust the constraint for them.
  //
  // TODO: Due to current inline asm didn't encourage to jmp to the outsider
  // label, so here we don't handle jmp function label now, but we need to
  // enhance it (especilly in PIC model) if we meet meaningful requirements.
  if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
      TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
      TM.getCodeModel() != CodeModel::Large) {
    OpInfo.isIndirect = false;
    OpInfo.ConstraintType = TargetLowering::C_Address;
  }

  // If this is a memory input, and if the operand is not indirect, do what we
  // need to provide an address for the memory input.
  if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
      !OpInfo.isIndirect) {
    assert((OpInfo.isMultipleAlternative ||(static_cast <bool> ((OpInfo.isMultipleAlternative || (
OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!"
) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8997
, __extension__ __PRETTY_FUNCTION__))
            (OpInfo.Type == InlineAsm::isInput)) &&(static_cast <bool> ((OpInfo.isMultipleAlternative || (
OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!"
) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8997
, __extension__ __PRETTY_FUNCTION__))
           "Can only indirectify direct input operands!")(static_cast <bool> ((OpInfo.isMultipleAlternative || (
OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!"
) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 8997
, __extension__ __PRETTY_FUNCTION__));

    // Memory operands really want the address of the value.
    Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);

    // There is no longer a Value* corresponding to this operand.
    OpInfo.CallOperandVal = nullptr;

    // It is now an indirect operand.
    OpInfo.isIndirect = true;
  }

}

// AsmNodeOperands - The operands for the ISD::INLINEASM node.
std::vector<SDValue> AsmNodeOperands;
AsmNodeOperands.push_back(SDValue());  // reserve space for input chain
AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
    IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));

// If we have a !srcloc metadata node associated with it, we want to attach
// this to the ultimately generated inline asm machineinstr.  To do this, we
// pass in the third operand as this (potentially null) inline asm MDNode.
const MDNode *SrcLoc = Call.getMetadata("srcloc");
AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));

// Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
// bits as operand 3.
AsmNodeOperands.push_back(DAG.getTargetConstant(
    ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));

// Third pass: Loop over operands to prepare DAG-level operands.. As part of
// this, assign virtual and physical registers for inputs and otput.
for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
9
←
Assuming '__begin1' is equal to '__end1'→
  // Assign Registers.
  SDISelAsmOperandInfo &RefOpInfo =
      OpInfo.isMatchingInputConstraint()
          ? ConstraintOperands[OpInfo.getMatchedOperand()]
          : OpInfo;
  const auto RegError =
      getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
  if (RegError) {
    const MachineFunction &MF = DAG.getMachineFunction();
    const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
    const char *RegName = TRI.getName(*RegError);
    emitInlineAsmError(Call, "register '" + Twine(RegName) +
                                 "' allocated for constraint '" +
                                 Twine(OpInfo.ConstraintCode) +
                                 "' does not match required type");
    return;
  }

  auto DetectWriteToReservedRegister = [&]() {
    const MachineFunction &MF = DAG.getMachineFunction();
    const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
    for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
      if (Register::isPhysicalRegister(Reg) &&
          TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
        const char *RegName = TRI.getName(Reg);
        emitInlineAsmError(Call, "write to reserved register '" +
                                     Twine(RegName) + "'");
        return true;
      }
    }
    return false;
  };
  assert((OpInfo.ConstraintType != TargetLowering::C_Address ||(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering
::C_Address || (OpInfo.Type == InlineAsm::isInput && !
OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed."
) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066
, __extension__ __PRETTY_FUNCTION__))
          (OpInfo.Type == InlineAsm::isInput &&(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering
::C_Address || (OpInfo.Type == InlineAsm::isInput && !
OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed."
) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066
, __extension__ __PRETTY_FUNCTION__))
           !OpInfo.isMatchingInputConstraint())) &&(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering
::C_Address || (OpInfo.Type == InlineAsm::isInput && !
OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed."
) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066
, __extension__ __PRETTY_FUNCTION__))
         "Only address as input operand is allowed.")(static_cast <bool> ((OpInfo.ConstraintType != TargetLowering
::C_Address || (OpInfo.Type == InlineAsm::isInput && !
OpInfo.isMatchingInputConstraint())) && "Only address as input operand is allowed."
) ? void (0) : __assert_fail ("(OpInfo.ConstraintType != TargetLowering::C_Address || (OpInfo.Type == InlineAsm::isInput && !OpInfo.isMatchingInputConstraint())) && \"Only address as input operand is allowed.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9066
, __extension__ __PRETTY_FUNCTION__));

  switch (OpInfo.Type) {
  case InlineAsm::isOutput:
    if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
      unsigned ConstraintID =
          TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
      assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
 && "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9074
, __extension__ __PRETTY_FUNCTION__))
             "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
 && "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9074
, __extension__ __PRETTY_FUNCTION__));

      // Add information to the INLINEASM node to know about this output.
      unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
      OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
      AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
                                                      MVT::i32));
      AsmNodeOperands.push_back(OpInfo.CallOperand);
    } else {
      // Otherwise, this outputs to a register (directly for C_Register /
      // C_RegisterClass, and a target-defined fashion for
      // C_Immediate/C_Other). Find a register that we can use.
      if (OpInfo.AssignedRegs.Regs.empty()) {
        emitInlineAsmError(
            Call, "couldn't allocate output register for constraint '" +
                      Twine(OpInfo.ConstraintCode) + "'");
        return;
      }

      if (DetectWriteToReservedRegister())
        return;

      // Add information to the INLINEASM node to know that this register is
      // set.
      OpInfo.AssignedRegs.AddInlineAsmOperands(
          OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber
                                : InlineAsm::Kind_RegDef,
          false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
    }
    break;

  case InlineAsm::isInput:
  case InlineAsm::isLabel: {
    SDValue InOperandVal = OpInfo.CallOperand;

    if (OpInfo.isMatchingInputConstraint()) {
      // If this is required to match an output register we have already set,
      // just use its register.
      auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
                                                AsmNodeOperands);
      unsigned OpFlag =
        cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
      if (InlineAsm::isRegDefKind(OpFlag) ||
          InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
        // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
        if (OpInfo.isIndirect) {
          // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
          emitInlineAsmError(Call, "inline asm not supported yet: "
                                   "don't know how to handle tied "
                                   "indirect register inputs");
          return;
        }

        SmallVector<unsigned, 4> Regs;
        MachineFunction &MF = DAG.getMachineFunction();
        MachineRegisterInfo &MRI = MF.getRegInfo();
        const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
        auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
        Register TiedReg = R->getReg();
        MVT RegVT = R->getSimpleValueType(0);
        const TargetRegisterClass *RC =
            TiedReg.isVirtual()     ? MRI.getRegClass(TiedReg)
            : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
                                    : TRI.getMinimalPhysRegClass(TiedReg);
        unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
        for (unsigned i = 0; i != NumRegs; ++i)
          Regs.push_back(MRI.createVirtualRegister(RC));

        RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());

        SDLoc dl = getCurSDLoc();
        // Use the produced MatchedRegs object to
        MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
        MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
                                         true, OpInfo.getMatchedOperand(), dl,
                                         DAG, AsmNodeOperands);
        break;
      }

      assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!")(static_cast <bool> (InlineAsm::isMemKind(OpFlag) &&
 "Unknown matching constraint!") ? void (0) : __assert_fail (
"InlineAsm::isMemKind(OpFlag) && \"Unknown matching constraint!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9153
, __extension__ __PRETTY_FUNCTION__));
      assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&(static_cast <bool> (InlineAsm::getNumOperandRegisters(
OpFlag) == 1 && "Unexpected number of operands") ? void
 (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9155
, __extension__ __PRETTY_FUNCTION__))
             "Unexpected number of operands")(static_cast <bool> (InlineAsm::getNumOperandRegisters(
OpFlag) == 1 && "Unexpected number of operands") ? void
 (0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9155
, __extension__ __PRETTY_FUNCTION__));
      // Add information to the INLINEASM node to know about this input.
      // See InlineAsm.h isUseOperandTiedToDef.
      OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
      OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
                                                  OpInfo.getMatchedOperand());
      AsmNodeOperands.push_back(DAG.getTargetConstant(
          OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
      AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
      break;
    }

    // Treat indirect 'X' constraint as memory.
    if (OpInfo.ConstraintType == TargetLowering::C_Other &&
        OpInfo.isIndirect)
      OpInfo.ConstraintType = TargetLowering::C_Memory;

    if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
        OpInfo.ConstraintType == TargetLowering::C_Other) {
      std::vector<SDValue> Ops;
      TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
                                        Ops, DAG);
      if (Ops.empty()) {
        if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
          if (isa<ConstantSDNode>(InOperandVal)) {
            emitInlineAsmError(Call, "value out of range for constraint '" +
                                         Twine(OpInfo.ConstraintCode) + "'");
            return;
          }

        emitInlineAsmError(Call,
                           "invalid operand for inline asm constraint '" +
                               Twine(OpInfo.ConstraintCode) + "'");
        return;
      }

      // Add information to the INLINEASM node to know about this input.
      unsigned ResOpType =
        InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
      AsmNodeOperands.push_back(DAG.getTargetConstant(
          ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
      llvm::append_range(AsmNodeOperands, Ops);
      break;
    }

    if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
      assert((OpInfo.isIndirect ||(static_cast <bool> ((OpInfo.isIndirect || OpInfo.ConstraintType
 != TargetLowering::C_Memory) && "Operand must be indirect to be a mem!"
) ? void (0) : __assert_fail ("(OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && \"Operand must be indirect to be a mem!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9203
, __extension__ __PRETTY_FUNCTION__))
              OpInfo.ConstraintType != TargetLowering::C_Memory) &&(static_cast <bool> ((OpInfo.isIndirect || OpInfo.ConstraintType
 != TargetLowering::C_Memory) && "Operand must be indirect to be a mem!"
) ? void (0) : __assert_fail ("(OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && \"Operand must be indirect to be a mem!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9203
, __extension__ __PRETTY_FUNCTION__))
             "Operand must be indirect to be a mem!")(static_cast <bool> ((OpInfo.isIndirect || OpInfo.ConstraintType
 != TargetLowering::C_Memory) && "Operand must be indirect to be a mem!"
) ? void (0) : __assert_fail ("(OpInfo.isIndirect || OpInfo.ConstraintType != TargetLowering::C_Memory) && \"Operand must be indirect to be a mem!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9203
, __extension__ __PRETTY_FUNCTION__));
      assert(InOperandVal.getValueType() ==(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9206
, __extension__ __PRETTY_FUNCTION__))
                 TLI.getPointerTy(DAG.getDataLayout()) &&(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9206
, __extension__ __PRETTY_FUNCTION__))
             "Memory operands expect pointer values")(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9206
, __extension__ __PRETTY_FUNCTION__));

      unsigned ConstraintID =
          TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
      assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
 && "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9211
, __extension__ __PRETTY_FUNCTION__))
             "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
 && "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9211
, __extension__ __PRETTY_FUNCTION__));

      // Add information to the INLINEASM node to know about this input.
      unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
      ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
      AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
                                                      getCurSDLoc(),
                                                      MVT::i32));
      AsmNodeOperands.push_back(InOperandVal);
      break;
    }

    if (OpInfo.ConstraintType == TargetLowering::C_Address) {
      assert(InOperandVal.getValueType() ==(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Address operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Address operands expect pointer values\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9226
, __extension__ __PRETTY_FUNCTION__))
                 TLI.getPointerTy(DAG.getDataLayout()) &&(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Address operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Address operands expect pointer values\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9226
, __extension__ __PRETTY_FUNCTION__))
             "Address operands expect pointer values")(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Address operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Address operands expect pointer values\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9226
, __extension__ __PRETTY_FUNCTION__));

      unsigned ConstraintID =
          TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
      assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
 && "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9231
, __extension__ __PRETTY_FUNCTION__))
             "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
 && "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9231
, __extension__ __PRETTY_FUNCTION__));

      unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);

      SDValue AsmOp = InOperandVal;
      if (isFunction(InOperandVal)) {
        auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
        ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Func, 1);
        AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
                                           InOperandVal.getValueType(),
                                           GA->getOffset());
      }

      // Add information to the INLINEASM node to know about this input.
      ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);

      AsmNodeOperands.push_back(
          DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));

      AsmNodeOperands.push_back(AsmOp);
      break;
    }

    assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering
::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::
C_Register) && "Unknown constraint type!") ? void (0)
 : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9256
, __extension__ __PRETTY_FUNCTION__))
            OpInfo.ConstraintType == TargetLowering::C_Register) &&(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering
::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::
C_Register) && "Unknown constraint type!") ? void (0)
 : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9256
, __extension__ __PRETTY_FUNCTION__))
           "Unknown constraint type!")(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering
::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::
C_Register) && "Unknown constraint type!") ? void (0)
 : __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9256
, __extension__ __PRETTY_FUNCTION__));

    // TODO: Support this.
    if (OpInfo.isIndirect) {
      emitInlineAsmError(
          Call, "Don't know how to handle indirect register inputs yet "
                "for constraint '" +
                    Twine(OpInfo.ConstraintCode) + "'");
      return;
    }

    // Copy the input into the appropriate registers.
    if (OpInfo.AssignedRegs.Regs.empty()) {
      emitInlineAsmError(Call,
                         "couldn't allocate input reg for constraint '" +
                             Twine(OpInfo.ConstraintCode) + "'");
      return;
    }

    if (DetectWriteToReservedRegister())
      return;

    SDLoc dl = getCurSDLoc();

    OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
                                      &Call);

    OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
                                             dl, DAG, AsmNodeOperands);
    break;
  }
  case InlineAsm::isClobber:
    // Add the clobbered value to the operand list, so that the register
    // allocator is aware that the physreg got clobbered.
    if (!OpInfo.AssignedRegs.Regs.empty())
      OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
                                               false, 0, getCurSDLoc(), DAG,
                                               AsmNodeOperands);
    break;
  }
}

// Finish up input operands.  Set the input chain and add the flag last.
AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
if (Glue.getNode()) AsmNodeOperands.push_back(Glue);

unsigned ISDOpc = IsCallBr10.1
'IsCallBr' is false
1
'IsCallBr' is false
1
'IsCallBr' is false
 ? ISD::INLINEASM_BR : ISD::INLINEASM;
10
←
Taking false branch→
11
←
'?' condition is false→
Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
                    DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
Glue = Chain.getValue(1);

// Do additional work to generate outputs.

SmallVector<EVT, 1> ResultVTs;
SmallVector<SDValue, 1> ResultValues;
SmallVector<SDValue, 8> OutChains;

llvm::Type *CallResultType = Call.getType();
ArrayRef<Type *> ResultTypes;
if (StructType *StructResult12.1
'StructResult' is null
1
'StructResult' is null
1
'StructResult' is null
 = dyn_cast<StructType>(CallResultType))
12
←
Assuming 'CallResultType' is not a 'CastReturnType'→
13
←
Taking false branch→
  ResultTypes = StructResult->elements();
else if (!CallResultType->isVoidTy())
14
←
Taking true branch→
  ResultTypes = ArrayRef(CallResultType);

auto CurResultType = ResultTypes.begin();
auto handleRegAssign = [&](SDValue V) {
  assert(CurResultType != ResultTypes.end() && "Unexpected value")(static_cast <bool> (CurResultType != ResultTypes.end()
 && "Unexpected value") ? void (0) : __assert_fail ("CurResultType != ResultTypes.end() && \"Unexpected value\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9322
, __extension__ __PRETTY_FUNCTION__));
  assert((*CurResultType)->isSized() && "Unexpected unsized type")(static_cast <bool> ((*CurResultType)->isSized() &&
 "Unexpected unsized type") ? void (0) : __assert_fail ("(*CurResultType)->isSized() && \"Unexpected unsized type\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9323
, __extension__ __PRETTY_FUNCTION__));
  EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
  ++CurResultType;
  // If the type of the inline asm call site return value is different but has
  // same size as the type of the asm output bitcast it.  One example of this
  // is for vectors with different width / number of elements.  This can
  // happen for register classes that can contain multiple different value
  // types.  The preg or vreg allocated may not have the same VT as was
  // expected.
  //
  // This can also happen for a return value that disagrees with the register
  // class it is put in, eg. a double in a general-purpose register on a
  // 32-bit machine.
  if (ResultVT != V.getValueType() &&
      ResultVT.getSizeInBits() == V.getValueSizeInBits())
    V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
  else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
           V.getValueType().isInteger()) {
    // If a result value was tied to an input value, the computed result
    // may have a wider width than the expected result.  Extract the
    // relevant portion.
    V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
  }
  assert(ResultVT == V.getValueType() && "Asm result value mismatch!")(static_cast <bool> (ResultVT == V.getValueType() &&
 "Asm result value mismatch!") ? void (0) : __assert_fail ("ResultVT == V.getValueType() && \"Asm result value mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9346
, __extension__ __PRETTY_FUNCTION__));
  ResultVTs.push_back(ResultVT);
  ResultValues.push_back(V);
};

// Deal with output operands.
for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
15
←
Assuming '__begin1' is not equal to '__end1'→
  if (OpInfo.Type == InlineAsm::isOutput) {
16
←
Assuming field 'Type' is equal to isOutput→
17
←
Taking true branch→
    SDValue Val;
18
←
Calling defaulted default constructor for 'SDValue'→
20
←
Returning from default constructor for 'SDValue'→
21
←
'Val' initialized here→
    // Skip trivial output operands.
    if (OpInfo.AssignedRegs.Regs.empty())
22
←
Taking false branch→
      continue;

    switch (OpInfo.ConstraintType) {
23
←
Control jumps to 'case C_Address:'  at line 9372→
    case TargetLowering::C_Register:
    case TargetLowering::C_RegisterClass:
      Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
                                                Chain, &Glue, &Call);
      break;
    case TargetLowering::C_Immediate:
    case TargetLowering::C_Other:
      Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
                                            OpInfo, DAG);
      break;
    case TargetLowering::C_Memory:
      break; // Already handled.
    case TargetLowering::C_Address:
      break; // Silence warning.
    case TargetLowering::C_Unknown:
      assert(false && "Unexpected unknown constraint")(static_cast <bool> (false && "Unexpected unknown constraint"
) ? void (0) : __assert_fail ("false && \"Unexpected unknown constraint\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9375
, __extension__ __PRETTY_FUNCTION__));
    }

    // Indirect output manifest as stores. Record output chains.
    if (OpInfo.isIndirect) {
24
←
 Execution continues on line 9379→
25
←
Assuming field 'isIndirect' is true→
26
←
Taking true branch→
      const Value *Ptr = OpInfo.CallOperandVal;
      assert(Ptr && "Expected value CallOperandVal for indirect asm operand")(static_cast <bool> (Ptr && "Expected value CallOperandVal for indirect asm operand"
) ? void (0) : __assert_fail ("Ptr && \"Expected value CallOperandVal for indirect asm operand\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9381
, __extension__ __PRETTY_FUNCTION__));
27
←
Assuming 'Ptr' is non-null→
28
←
'?' condition is true→
      SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
29
←
Null pointer value stored to 'Val.Node'→
30
←
Calling 'SelectionDAG::getStore'→
                                   MachinePointerInfo(Ptr));
      OutChains.push_back(Store);
    } else {
      // generate CopyFromRegs to associated registers.
      assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
 "Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9387
, __extension__ __PRETTY_FUNCTION__));
      if (Val.getOpcode() == ISD::MERGE_VALUES) {
        for (const SDValue &V : Val->op_values())
          handleRegAssign(V);
      } else
        handleRegAssign(Val);
    }
  }
}

// Set results.
if (!ResultValues.empty()) {
  assert(CurResultType == ResultTypes.end() &&(static_cast <bool> (CurResultType == ResultTypes.end()
 && "Mismatch in number of ResultTypes") ? void (0) :
 __assert_fail ("CurResultType == ResultTypes.end() && \"Mismatch in number of ResultTypes\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9400
, __extension__ __PRETTY_FUNCTION__))
         "Mismatch in number of ResultTypes")(static_cast <bool> (CurResultType == ResultTypes.end()
 && "Mismatch in number of ResultTypes") ? void (0) :
 __assert_fail ("CurResultType == ResultTypes.end() && \"Mismatch in number of ResultTypes\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9400
, __extension__ __PRETTY_FUNCTION__));
  assert(ResultValues.size() == ResultTypes.size() &&(static_cast <bool> (ResultValues.size() == ResultTypes
.size() && "Mismatch in number of output operands in asm result"
) ? void (0) : __assert_fail ("ResultValues.size() == ResultTypes.size() && \"Mismatch in number of output operands in asm result\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9402
, __extension__ __PRETTY_FUNCTION__))
         "Mismatch in number of output operands in asm result")(static_cast <bool> (ResultValues.size() == ResultTypes
.size() && "Mismatch in number of output operands in asm result"
) ? void (0) : __assert_fail ("ResultValues.size() == ResultTypes.size() && \"Mismatch in number of output operands in asm result\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9402
, __extension__ __PRETTY_FUNCTION__));

  SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                          DAG.getVTList(ResultVTs), ResultValues);
  setValue(&Call, V);
}

// Collect store chains.
if (!OutChains.empty())
  Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);

if (EmitEHLabels) {
  Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
}

// Only Update Root if inline assembly has a memory effect.
if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
    EmitEHLabels)
  DAG.setRoot(Chain);
9421}

9423void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
                                           const Twine &Message) {
LLVMContext &Ctx = *DAG.getContext();
Ctx.emitError(&Call, Message);

// Make sure we leave the DAG in a valid state
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SmallVector<EVT, 1> ValueVTs;
ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);

if (ValueVTs.empty())
  return;

SmallVector<SDValue, 1> Ops;
for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
  Ops.push_back(DAG.getUNDEF(ValueVTs[i]));

setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
9441}

9443void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
                        MVT::Other, getRoot(),
                        getValue(I.getArgOperand(0)),
                        DAG.getSrcValue(I.getArgOperand(0))));
9448}

9450void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const DataLayout &DL = DAG.getDataLayout();
SDValue V = DAG.getVAArg(
    TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
    getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
    DL.getABITypeAlign(I.getType()).value());
DAG.setRoot(V.getValue(1));

if (I.getType()->isPointerTy())
  V = DAG.getPtrExtOrTrunc(
      V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
setValue(&I, V);
9463}

9465void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
                        MVT::Other, getRoot(),
                        getValue(I.getArgOperand(0)),
                        DAG.getSrcValue(I.getArgOperand(0))));
9470}

9472void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
                        MVT::Other, getRoot(),
                        getValue(I.getArgOperand(0)),
                        getValue(I.getArgOperand(1)),
                        DAG.getSrcValue(I.getArgOperand(0)),
                        DAG.getSrcValue(I.getArgOperand(1))));
9479}

9481SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
                                                  const Instruction &I,
                                                  SDValue Op) {
const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
if (!Range)
  return Op;

ConstantRange CR = getConstantRangeFromMetadata(*Range);
if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped())
  return Op;

APInt Lo = CR.getUnsignedMin();
if (!Lo.isMinValue())
  return Op;

APInt Hi = CR.getUnsignedMax();
unsigned Bits = std::max(Hi.getActiveBits(),
                         static_cast<unsigned>(IntegerType::MIN_INT_BITS));

EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);

SDLoc SL = getCurSDLoc();

SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
                           DAG.getValueType(SmallVT));
unsigned NumVals = Op.getNode()->getNumValues();
if (NumVals == 1)
  return ZExt;

SmallVector<SDValue, 4> Ops;

Ops.push_back(ZExt);
for (unsigned I = 1; I != NumVals; ++I)
  Ops.push_back(Op.getValue(I));

return DAG.getMergeValues(Ops, SL);
9517}

9519/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
9520/// the call being lowered.
9521///
9522/// This is a helper for lowering intrinsics that follow a target calling
9523/// convention or require stack pointer adjustment. Only a subset of the
9524/// intrinsic's operands need to participate in the calling convention.
9525void SelectionDAGBuilder::populateCallLoweringInfo(
  TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
  unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
  bool IsPatchPoint) {
TargetLowering::ArgListTy Args;
Args.reserve(NumArgs);

// Populate the argument list.
// Attributes for args start at offset 1, after the return attribute.
for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
     ArgI != ArgE; ++ArgI) {
  const Value *V = Call->getOperand(ArgI);

  assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.")(static_cast <bool> (!V->getType()->isEmptyTy() &&
 "Empty type passed to intrinsic.") ? void (0) : __assert_fail
 ("!V->getType()->isEmptyTy() && \"Empty type passed to intrinsic.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9538
, __extension__ __PRETTY_FUNCTION__));

  TargetLowering::ArgListEntry Entry;
  Entry.Node = getValue(V);
  Entry.Ty = V->getType();
  Entry.setAttributes(Call, ArgI);
  Args.push_back(Entry);
}

CLI.setDebugLoc(getCurSDLoc())
    .setChain(getRoot())
    .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
    .setDiscardResult(Call->use_empty())
    .setIsPatchPoint(IsPatchPoint)
    .setIsPreallocated(
        Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
9554}

9556/// Add a stack map intrinsic call's live variable operands to a stackmap
9557/// or patchpoint target node's operand list.
9558///
9559/// Constants are converted to TargetConstants purely as an optimization to
9560/// avoid constant materialization and register allocation.
9561///
9562/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
9563/// generate addess computation nodes, and so FinalizeISel can convert the
9564/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
9565/// address materialization and register allocation, but may also be required
9566/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
9567/// alloca in the entry block, then the runtime may assume that the alloca's
9568/// StackMap location can be read immediately after compilation and that the
9569/// location is valid at any point during execution (this is similar to the
9570/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
9571/// only available in a register, then the runtime would need to trap when
9572/// execution reaches the StackMap in order to read the alloca's location.
9573static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
                              const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
                              SelectionDAGBuilder &Builder) {
SelectionDAG &DAG = Builder.DAG;
for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
  SDValue Op = Builder.getValue(Call.getArgOperand(I));

  // Things on the stack are pointer-typed, meaning that they are already
  // legal and can be emitted directly to target nodes.
  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
    Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
  } else {
    // Otherwise emit a target independent node to be legalised.
    Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
  }
}
9589}

9591/// Lower llvm.experimental.stackmap.
9592void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
// void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
//                                  [live variables...])

assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.")(static_cast <bool> (CI.getType()->isVoidTy() &&
 "Stackmap cannot return a value.") ? void (0) : __assert_fail
 ("CI.getType()->isVoidTy() && \"Stackmap cannot return a value.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9596
, __extension__ __PRETTY_FUNCTION__));

SDValue Chain, InGlue, Callee;
SmallVector<SDValue, 32> Ops;

SDLoc DL = getCurSDLoc();
Callee = getValue(CI.getCalledOperand());

// The stackmap intrinsic only records the live variables (the arguments
// passed to it) and emits NOPS (if requested). Unlike the patchpoint
// intrinsic, this won't be lowered to a function call. This means we don't
// have to worry about calling conventions and target specific lowering code.
// Instead we perform the call lowering right here.
//
// chain, flag = CALLSEQ_START(chain, 0, 0)
// chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
// chain, flag = CALLSEQ_END(chain, 0, 0, flag)
//
Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
InGlue = Chain.getValue(1);

// Add the STACKMAP operands, starting with DAG house-keeping.
Ops.push_back(Chain);
Ops.push_back(InGlue);

// Add the <id>, <numShadowBytes> operands.
//
// These do not require legalisation, and can be emitted directly to target
// constant nodes.
SDValue ID = getValue(CI.getArgOperand(0));
assert(ID.getValueType() == MVT::i64)(static_cast <bool> (ID.getValueType() == MVT::i64) ? void
 (0) : __assert_fail ("ID.getValueType() == MVT::i64", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9626, __extension__ __PRETTY_FUNCTION__));
SDValue IDConst = DAG.getTargetConstant(
    cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType());
Ops.push_back(IDConst);

SDValue Shad = getValue(CI.getArgOperand(1));
assert(Shad.getValueType() == MVT::i32)(static_cast <bool> (Shad.getValueType() == MVT::i32) ?
 void (0) : __assert_fail ("Shad.getValueType() == MVT::i32",
 "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9632
, __extension__ __PRETTY_FUNCTION__));
SDValue ShadConst = DAG.getTargetConstant(
    cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType());
Ops.push_back(ShadConst);

// Add the live variables.
addStackMapLiveVars(CI, 2, DL, Ops, *this);

// Create the STACKMAP node.
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
InGlue = Chain.getValue(1);

Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);

// Stackmaps don't generate values, so nothing goes into the NodeMap.

// Set the root to the target-lowered call chain.
DAG.setRoot(Chain);

// Inform the Frame Information that we have a stackmap in this function.
FuncInfo.MF->getFrameInfo().setHasStackMap();
9654}

9656/// Lower llvm.experimental.patchpoint directly to its target opcode.
9657void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
                                        const BasicBlock *EHPadBB) {
// void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
//                                                 i32 <numBytes>,
//                                                 i8* <target>,
//                                                 i32 <numArgs>,
//                                                 [Args...],
//                                                 [live variables...])

CallingConv::ID CC = CB.getCallingConv();
bool IsAnyRegCC = CC == CallingConv::AnyReg;
bool HasDef = !CB.getType()->isVoidTy();
SDLoc dl = getCurSDLoc();
SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos));

// Handle immediate and symbolic callees.
if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
  Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
                                 /*isTarget=*/true);
else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
  Callee =  DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
                                       SDLoc(SymbolicCallee),
                                       SymbolicCallee->getValueType(0));

// Get the real number of arguments participating in the call <numArgs>
SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos));
unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();

// Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
// Intrinsics include all meta-operands up to but not including CC.
unsigned NumMetaOpers = PatchPointOpers::CCPos;
assert(CB.arg_size() >= NumMetaOpers + NumArgs &&(static_cast <bool> (CB.arg_size() >= NumMetaOpers +
 NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? void (0) : __assert_fail ("CB.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9689
, __extension__ __PRETTY_FUNCTION__))
       "Not enough arguments provided to the patchpoint intrinsic")(static_cast <bool> (CB.arg_size() >= NumMetaOpers +
 NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? void (0) : __assert_fail ("CB.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9689
, __extension__ __PRETTY_FUNCTION__));

// For AnyRegCC the arguments are lowered later on manually.
unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
Type *ReturnTy =
    IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();

TargetLowering::CallLoweringInfo CLI(DAG);
populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
                         ReturnTy, true);
std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);

SDNode *CallEnd = Result.second.getNode();
if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
  CallEnd = CallEnd->getOperand(0).getNode();

/// Get a call instruction from the call sequence chain.
/// Tail calls are not allowed.
assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END
 && "Expected a callseq node.") ? void (0) : __assert_fail
 ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9708
, __extension__ __PRETTY_FUNCTION__))
       "Expected a callseq node.")(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END
 && "Expected a callseq node.") ? void (0) : __assert_fail
 ("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9708
, __extension__ __PRETTY_FUNCTION__));
SDNode *Call = CallEnd->getOperand(0).getNode();
bool HasGlue = Call->getGluedNode();

// Replace the target specific call node with the patchable intrinsic.
SmallVector<SDValue, 8> Ops;

// Push the chain.
Ops.push_back(*(Call->op_begin()));

// Optionally, push the glue (if any).
if (HasGlue)
  Ops.push_back(*(Call->op_end() - 1));

// Push the register mask info.
if (HasGlue)
  Ops.push_back(*(Call->op_end() - 2));
else
  Ops.push_back(*(Call->op_end() - 1));

// Add the <id> and <numBytes> constants.
SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos));
Ops.push_back(DAG.getTargetConstant(
                cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos));
Ops.push_back(DAG.getTargetConstant(
                cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
                MVT::i32));

// Add the callee.
Ops.push_back(Callee);

// Adjust <numArgs> to account for any arguments that have been passed on the
// stack instead.
// Call Node: Chain, Target, {Args}, RegMask, [Glue]
unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));

// Add the calling convention
Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));

// Add the arguments we omitted previously. The register allocator should
// place these in any free register.
if (IsAnyRegCC)
  for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
    Ops.push_back(getValue(CB.getArgOperand(i)));

// Push the arguments from the call instruction.
SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
Ops.append(Call->op_begin() + 2, e);

// Push live variables for the stack map.
addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);

SDVTList NodeTys;
if (IsAnyRegCC && HasDef) {
  // Create the return types based on the intrinsic definition
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  SmallVector<EVT, 3> ValueVTs;
  ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
  assert(ValueVTs.size() == 1 && "Expected only one return value type.")(static_cast <bool> (ValueVTs.size() == 1 && "Expected only one return value type."
) ? void (0) : __assert_fail ("ValueVTs.size() == 1 && \"Expected only one return value type.\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9769
, __extension__ __PRETTY_FUNCTION__));

  // There is always a chain and a glue type at the end
  ValueVTs.push_back(MVT::Other);
  ValueVTs.push_back(MVT::Glue);
  NodeTys = DAG.getVTList(ValueVTs);
} else
  NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);

// Replace the target specific call node with a PATCHPOINT node.
SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);

// Update the NodeMap.
if (HasDef) {
  if (IsAnyRegCC)
    setValue(&CB, SDValue(PPV.getNode(), 0));
  else
    setValue(&CB, Result.first);
}

// Fixup the consumers of the intrinsic. The chain and glue may be used in the
// call sequence. Furthermore the location of the chain and glue can change
// when the AnyReg calling convention is used and the intrinsic returns a
// value.
if (IsAnyRegCC && HasDef) {
  SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
  SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
  DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
} else
  DAG.ReplaceAllUsesWith(Call, PPV.getNode());
DAG.DeleteNode(Call);

// Inform the Frame Information that we have a patchpoint in this function.
FuncInfo.MF->getFrameInfo().setHasPatchPoint();
9803}

9805void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
                                          unsigned Intrinsic) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
SDValue Op1 = getValue(I.getArgOperand(0));
SDValue Op2;
if (I.arg_size() > 1)
  Op2 = getValue(I.getArgOperand(1));
SDLoc dl = getCurSDLoc();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
SDValue Res;
SDNodeFlags SDFlags;
if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
  SDFlags.copyFMF(*FPMO);

switch (Intrinsic) {
case Intrinsic::vector_reduce_fadd:
  if (SDFlags.hasAllowReassociation())
    Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
                      DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
                      SDFlags);
  else
    Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
  break;
case Intrinsic::vector_reduce_fmul:
  if (SDFlags.hasAllowReassociation())
    Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
                      DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
                      SDFlags);
  else
    Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
  break;
case Intrinsic::vector_reduce_add:
  Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_mul:
  Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_and:
  Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_or:
  Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_xor:
  Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_smax:
  Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_smin:
  Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_umax:
  Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_umin:
  Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
  break;
case Intrinsic::vector_reduce_fmax:
  Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
  break;
case Intrinsic::vector_reduce_fmin:
  Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
  break;
default:
  llvm_unreachable("Unhandled vector reduce intrinsic")::llvm::llvm_unreachable_internal("Unhandled vector reduce intrinsic"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 9870
);
}
setValue(&I, Res);
9873}

9875/// Returns an AttributeList representing the attributes applied to the return
9876/// value of the given call.
9877static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
SmallVector<Attribute::AttrKind, 2> Attrs;
if (CLI.RetSExt)
  Attrs.push_back(Attribute::SExt);
if (CLI.RetZExt)
  Attrs.push_back(Attribute::ZExt);
if (CLI.IsInReg)
  Attrs.push_back(Attribute::InReg);

return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
                          Attrs);
9888}

9890/// TargetLowering::LowerCallTo - This is the default LowerCallTo
9891/// implementation, which just calls LowerCall.
9892/// FIXME: When all targets are
9893/// migrated to using LowerCall, this hook should be integrated into SDISel.
9894std::pair<SDValue, SDValue>
9895TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
// Handle the incoming return values from the call.
CLI.Ins.clear();
Type *OrigRetTy = CLI.RetTy;
SmallVector<EVT, 4> RetTys;
SmallVector<uint64_t, 4> Offsets;
auto &DL = CLI.DAG.getDataLayout();
ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);

if (CLI.IsPostTypeLegalization) {
  // If we are lowering a libcall after legalization, split the return type.
  SmallVector<EVT, 4> OldRetTys;
  SmallVector<uint64_t, 4> OldOffsets;
  RetTys.swap(OldRetTys);
  Offsets.swap(OldOffsets);

  for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
    EVT RetVT = OldRetTys[i];
    uint64_t Offset = OldOffsets[i];
    MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
    unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
    unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
    RetTys.append(NumRegs, RegisterVT);
    for (unsigned j = 0; j != NumRegs; ++j)
      Offsets.push_back(Offset + j * RegisterVTByteSZ);
  }
}

SmallVector<ISD::OutputArg, 4> Outs;
GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);

bool CanLowerReturn =
    this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
                         CLI.IsVarArg, Outs, CLI.RetTy->getContext());

SDValue DemoteStackSlot;
int DemoteStackIdx = -100;
if (!CanLowerReturn) {
  // FIXME: equivalent assert?
  // assert(!CS.hasInAllocaArgument() &&
  //        "sret demotion is incompatible with inalloca");
  uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
  Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
  MachineFunction &MF = CLI.DAG.getMachineFunction();
  DemoteStackIdx =
      MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
  Type *StackSlotPtrType = PointerType::get(CLI.RetTy,
                                            DL.getAllocaAddrSpace());

  DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
  ArgListEntry Entry;
  Entry.Node = DemoteStackSlot;
  Entry.Ty = StackSlotPtrType;
  Entry.IsSExt = false;
  Entry.IsZExt = false;
  Entry.IsInReg = false;
  Entry.IsSRet = true;
  Entry.IsNest = false;
  Entry.IsByVal = false;
  Entry.IsByRef = false;
  Entry.IsReturned = false;
  Entry.IsSwiftSelf = false;
  Entry.IsSwiftAsync = false;
  Entry.IsSwiftError = false;
  Entry.IsCFGuardTarget = false;
  Entry.Alignment = Alignment;
  CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
  CLI.NumFixedArgs += 1;
  CLI.getArgs()[0].IndirectType = CLI.RetTy;
  CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());

  // sret demotion isn't compatible with tail-calls, since the sret argument
  // points into the callers stack frame.
  CLI.IsTailCall = false;
} else {
  bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
      CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
    ISD::ArgFlagsTy Flags;
    if (NeedsRegBlock) {
      Flags.setInConsecutiveRegs();
      if (I == RetTys.size() - 1)
        Flags.setInConsecutiveRegsLast();
    }
    EVT VT = RetTys[I];
    MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
                                                   CLI.CallConv, VT);
    unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
                                                     CLI.CallConv, VT);
    for (unsigned i = 0; i != NumRegs; ++i) {
      ISD::InputArg MyFlags;
      MyFlags.Flags = Flags;
      MyFlags.VT = RegisterVT;
      MyFlags.ArgVT = VT;
      MyFlags.Used = CLI.IsReturnValueUsed;
      if (CLI.RetTy->isPointerTy()) {
        MyFlags.Flags.setPointer();
        MyFlags.Flags.setPointerAddrSpace(
            cast<PointerType>(CLI.RetTy)->getAddressSpace());
      }
      if (CLI.RetSExt)
        MyFlags.Flags.setSExt();
      if (CLI.RetZExt)
        MyFlags.Flags.setZExt();
      if (CLI.IsInReg)
        MyFlags.Flags.setInReg();
      CLI.Ins.push_back(MyFlags);
    }
  }
}

// We push in swifterror return as the last element of CLI.Ins.
ArgListTy &Args = CLI.getArgs();
if (supportSwiftError()) {
  for (const ArgListEntry &Arg : Args) {
    if (Arg.IsSwiftError) {
      ISD::InputArg MyFlags;
      MyFlags.VT = getPointerTy(DL);
      MyFlags.ArgVT = EVT(getPointerTy(DL));
      MyFlags.Flags.setSwiftError();
      CLI.Ins.push_back(MyFlags);
    }
  }
}

// Handle all of the outgoing arguments.
CLI.Outs.clear();
CLI.OutVals.clear();
for (unsigned i = 0, e = Args.size(); i != e; ++i) {
  SmallVector<EVT, 4> ValueVTs;
  ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
  // FIXME: Split arguments if CLI.IsPostTypeLegalization
  Type *FinalType = Args[i].Ty;
  if (Args[i].IsByVal)
    FinalType = Args[i].IndirectType;
  bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
      FinalType, CLI.CallConv, CLI.IsVarArg, DL);
  for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
       ++Value) {
    EVT VT = ValueVTs[Value];
    Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
    SDValue Op = SDValue(Args[i].Node.getNode(),
                         Args[i].Node.getResNo() + Value);
    ISD::ArgFlagsTy Flags;

    // Certain targets (such as MIPS), may have a different ABI alignment
    // for a type depending on the context. Give the target a chance to
    // specify the alignment it wants.
    const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
    Flags.setOrigAlign(OriginalAlignment);

    if (Args[i].Ty->isPointerTy()) {
      Flags.setPointer();
      Flags.setPointerAddrSpace(
          cast<PointerType>(Args[i].Ty)->getAddressSpace());
    }
    if (Args[i].IsZExt)
      Flags.setZExt();
    if (Args[i].IsSExt)
      Flags.setSExt();
    if (Args[i].IsInReg) {
      // If we are using vectorcall calling convention, a structure that is
      // passed InReg - is surely an HVA
      if (CLI.CallConv == CallingConv::X86_VectorCall &&
          isa<StructType>(FinalType)) {
        // The first value of a structure is marked
        if (0 == Value)
          Flags.setHvaStart();
        Flags.setHva();
      }
      // Set InReg Flag
      Flags.setInReg();
    }
    if (Args[i].IsSRet)
      Flags.setSRet();
    if (Args[i].IsSwiftSelf)
      Flags.setSwiftSelf();
    if (Args[i].IsSwiftAsync)
      Flags.setSwiftAsync();
    if (Args[i].IsSwiftError)
      Flags.setSwiftError();
    if (Args[i].IsCFGuardTarget)
      Flags.setCFGuardTarget();
    if (Args[i].IsByVal)
      Flags.setByVal();
    if (Args[i].IsByRef)
      Flags.setByRef();
    if (Args[i].IsPreallocated) {
      Flags.setPreallocated();
      // Set the byval flag for CCAssignFn callbacks that don't know about
      // preallocated.  This way we can know how many bytes we should've
      // allocated and how many bytes a callee cleanup function will pop.  If
      // we port preallocated to more targets, we'll have to add custom
      // preallocated handling in the various CC lowering callbacks.
      Flags.setByVal();
    }
    if (Args[i].IsInAlloca) {
      Flags.setInAlloca();
      // Set the byval flag for CCAssignFn callbacks that don't know about
      // inalloca.  This way we can know how many bytes we should've allocated
      // and how many bytes a callee cleanup function will pop.  If we port
      // inalloca to more targets, we'll have to add custom inalloca handling
      // in the various CC lowering callbacks.
      Flags.setByVal();
    }
    Align MemAlign;
    if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
      unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
      Flags.setByValSize(FrameSize);

      // info is not there but there are cases it cannot get right.
      if (auto MA = Args[i].Alignment)
        MemAlign = *MA;
      else
        MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL));
    } else if (auto MA = Args[i].Alignment) {
      MemAlign = *MA;
    } else {
      MemAlign = OriginalAlignment;
    }
    Flags.setMemAlign(MemAlign);
    if (Args[i].IsNest)
      Flags.setNest();
    if (NeedsRegBlock)
      Flags.setInConsecutiveRegs();

    MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
                                               CLI.CallConv, VT);
    unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
                                                      CLI.CallConv, VT);
    SmallVector<SDValue, 4> Parts(NumParts);
    ISD::NodeType ExtendKind = ISD::ANY_EXTEND;

    if (Args[i].IsSExt)
      ExtendKind = ISD::SIGN_EXTEND;
    else if (Args[i].IsZExt)
      ExtendKind = ISD::ZERO_EXTEND;

    // Conservatively only handle 'returned' on non-vectors that can be lowered,
    // for now.
    if (Args[i].IsReturned && !Op.getValueType().isVector() &&
        CanLowerReturn) {
      assert((CLI.RetTy == Args[i].Ty ||(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy
->isPointerTy() && Args[i].Ty->isPointerTy() &&
 CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace
())) && RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141
, __extension__ __PRETTY_FUNCTION__))
              (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy
->isPointerTy() && Args[i].Ty->isPointerTy() &&
 CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace
())) && RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141
, __extension__ __PRETTY_FUNCTION__))
               CLI.RetTy->getPointerAddressSpace() ==(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy
->isPointerTy() && Args[i].Ty->isPointerTy() &&
 CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace
())) && RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141
, __extension__ __PRETTY_FUNCTION__))
                   Args[i].Ty->getPointerAddressSpace())) &&(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy
->isPointerTy() && Args[i].Ty->isPointerTy() &&
 CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace
())) && RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141
, __extension__ __PRETTY_FUNCTION__))
             RetTys.size() == NumValues && "unexpected use of 'returned'")(static_cast <bool> ((CLI.RetTy == Args[i].Ty || (CLI.RetTy
->isPointerTy() && Args[i].Ty->isPointerTy() &&
 CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace
())) && RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("(CLI.RetTy == Args[i].Ty || (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && CLI.RetTy->getPointerAddressSpace() == Args[i].Ty->getPointerAddressSpace())) && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10141
, __extension__ __PRETTY_FUNCTION__));
      // Before passing 'returned' to the target lowering code, ensure that
      // either the register MVT and the actual EVT are the same size or that
      // the return value and argument are extended in the same way; in these
      // cases it's safe to pass the argument register value unchanged as the
      // return register value (although it's at the target's option whether
      // to do so)
      // TODO: allow code generation to take advantage of partially preserved
      // registers rather than clobbering the entire register when the
      // parameter extension method is not compatible with the return
      // extension method
      if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
          (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
           CLI.RetZExt == Args[i].IsZExt))
        Flags.setReturned();
    }

    getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
                   CLI.CallConv, ExtendKind);

    for (unsigned j = 0; j != NumParts; ++j) {
      // if it isn't first piece, alignment must be 1
      // For scalable vectors the scalable part is currently handled
      // by individual targets, so we just use the known minimum size here.
      ISD::OutputArg MyFlags(
          Flags, Parts[j].getValueType().getSimpleVT(), VT,
          i < CLI.NumFixedArgs, i,
          j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
      if (NumParts > 1 && j == 0)
        MyFlags.Flags.setSplit();
      else if (j != 0) {
        MyFlags.Flags.setOrigAlign(Align(1));
        if (j == NumParts - 1)
          MyFlags.Flags.setSplitEnd();
      }

      CLI.Outs.push_back(MyFlags);
      CLI.OutVals.push_back(Parts[j]);
    }

    if (NeedsRegBlock && Value == NumValues - 1)
      CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
  }
}

SmallVector<SDValue, 4> InVals;
CLI.Chain = LowerCall(CLI, InVals);

// Update CLI.InVals to use outside of this function.
CLI.InVals = InVals;

// Verify that the target's LowerCall behaved as expected.
assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&(static_cast <bool> (CLI.Chain.getNode() && CLI
.Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!"
) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10194
, __extension__ __PRETTY_FUNCTION__))
       "LowerCall didn't return a valid chain!")(static_cast <bool> (CLI.Chain.getNode() && CLI
.Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!"
) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10194
, __extension__ __PRETTY_FUNCTION__));
assert((!CLI.IsTailCall || InVals.empty()) &&(static_cast <bool> ((!CLI.IsTailCall || InVals.empty()
) && "LowerCall emitted a return value for a tail call!"
) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10196
, __extension__ __PRETTY_FUNCTION__))
       "LowerCall emitted a return value for a tail call!")(static_cast <bool> ((!CLI.IsTailCall || InVals.empty()
) && "LowerCall emitted a return value for a tail call!"
) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10196
, __extension__ __PRETTY_FUNCTION__));
assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&(static_cast <bool> ((CLI.IsTailCall || InVals.size() ==
 CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10198
, __extension__ __PRETTY_FUNCTION__))
       "LowerCall didn't emit the correct number of values!")(static_cast <bool> ((CLI.IsTailCall || InVals.size() ==
 CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10198
, __extension__ __PRETTY_FUNCTION__));

// For a tail call, the return value is merely live-out and there aren't
// any nodes in the DAG representing it. Return a special value to
// indicate that a tail call has been emitted and no more Instructions
// should be processed in the current block.
if (CLI.IsTailCall) {
  CLI.DAG.setRoot(CLI.Chain);
  return std::make_pair(SDValue(), SDValue());
}

10209#ifndef NDEBUG
for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
  assert(InVals[i].getNode() && "LowerCall emitted a null value!")(static_cast <bool> (InVals[i].getNode() && "LowerCall emitted a null value!"
) ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerCall emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10211
, __extension__ __PRETTY_FUNCTION__));
  assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType
() && "LowerCall emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10213
, __extension__ __PRETTY_FUNCTION__))
         "LowerCall emitted a value with the wrong type!")(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType
() && "LowerCall emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10213
, __extension__ __PRETTY_FUNCTION__));
}
10215#endif

SmallVector<SDValue, 4> ReturnValues;
if (!CanLowerReturn) {
  // The instruction result is the result of loading from the
  // hidden sret parameter.
  SmallVector<EVT, 1> PVTs;
  Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());

  ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
  assert(PVTs.size() == 1 && "Pointers should fit in one register")(static_cast <bool> (PVTs.size() == 1 && "Pointers should fit in one register"
) ? void (0) : __assert_fail ("PVTs.size() == 1 && \"Pointers should fit in one register\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10225
, __extension__ __PRETTY_FUNCTION__));
  EVT PtrVT = PVTs[0];

  unsigned NumValues = RetTys.size();
  ReturnValues.resize(NumValues);
  SmallVector<SDValue, 4> Chains(NumValues);

  // An aggregate return value cannot wrap around the address space, so
  // offsets to its parts don't wrap either.
  SDNodeFlags Flags;
  Flags.setNoUnsignedWrap(true);

  MachineFunction &MF = CLI.DAG.getMachineFunction();
  Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
  for (unsigned i = 0; i < NumValues; ++i) {
    SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
                                  CLI.DAG.getConstant(Offsets[i], CLI.DL,
                                                      PtrVT), Flags);
    SDValue L = CLI.DAG.getLoad(
        RetTys[i], CLI.DL, CLI.Chain, Add,
        MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
                                          DemoteStackIdx, Offsets[i]),
        HiddenSRetAlign);
    ReturnValues[i] = L;
    Chains[i] = L.getValue(1);
  }

  CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
} else {
  // Collect the legal value parts into potentially illegal values
  // that correspond to the original function's return values.
  std::optional<ISD::NodeType> AssertOp;
  if (CLI.RetSExt)
    AssertOp = ISD::AssertSext;
  else if (CLI.RetZExt)
    AssertOp = ISD::AssertZext;
  unsigned CurReg = 0;
  for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
    EVT VT = RetTys[I];
    MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
                                                   CLI.CallConv, VT);
    unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
                                                     CLI.CallConv, VT);

    ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
                                            NumRegs, RegisterVT, VT, nullptr,
                                            CLI.CallConv, AssertOp));
    CurReg += NumRegs;
  }

  // For a function returning void, there is no return value. We can't create
  // such a node, so we just return a null return value in that case. In
  // that case, nothing will actually look at the value.
  if (ReturnValues.empty())
    return std::make_pair(SDValue(), CLI.Chain);
}

SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
                              CLI.DAG.getVTList(RetTys), ReturnValues);
return std::make_pair(Res, CLI.Chain);
10285}

10287/// Places new result values for the node in Results (their number
10288/// and types must exactly match those of the original return values of
10289/// the node), or leaves Results empty, which indicates that the node is not
10290/// to be custom lowered after all.
10291void TargetLowering::LowerOperationWrapper(SDNode *N,
                                         SmallVectorImpl<SDValue> &Results,
                                         SelectionDAG &DAG) const {
SDValue Res = LowerOperation(SDValue(N, 0), DAG);

if (!Res.getNode())
  return;

// If the original node has one result, take the return value from
// LowerOperation as is. It might not be result number 0.
if (N->getNumValues() == 1) {
  Results.push_back(Res);
  return;
}

// If the original node has multiple results, then the return node should
// have the same number of results.
assert((N->getNumValues() == Res->getNumValues()) &&(static_cast <bool> ((N->getNumValues() == Res->getNumValues
()) && "Lowering returned the wrong number of results!"
) ? void (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10309
, __extension__ __PRETTY_FUNCTION__))
    "Lowering returned the wrong number of results!")(static_cast <bool> ((N->getNumValues() == Res->getNumValues
()) && "Lowering returned the wrong number of results!"
) ? void (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10309
, __extension__ __PRETTY_FUNCTION__));

// Places new result values base on N result number.
for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
  Results.push_back(Res.getValue(I));
10314}

10316SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
llvm_unreachable("LowerOperation not implemented for this target!")::llvm::llvm_unreachable_internal("LowerOperation not implemented for this target!"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10317
);
10318}

10320void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V,
                                                   unsigned Reg,
                                                   ISD::NodeType ExtendType) {
SDValue Op = getNonRegisterValue(V);
assert((Op.getOpcode() != ISD::CopyFromReg ||(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg
 || cast<RegisterSDNode>(Op.getOperand(1))->getReg()
 != Reg) && "Copy from a reg to the same reg!") ? void
 (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10326
, __extension__ __PRETTY_FUNCTION__))
        cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg
 || cast<RegisterSDNode>(Op.getOperand(1))->getReg()
 != Reg) && "Copy from a reg to the same reg!") ? void
 (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10326
, __extension__ __PRETTY_FUNCTION__))
       "Copy from a reg to the same reg!")(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg
 || cast<RegisterSDNode>(Op.getOperand(1))->getReg()
 != Reg) && "Copy from a reg to the same reg!") ? void
 (0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10326
, __extension__ __PRETTY_FUNCTION__));
assert(!Register::isPhysicalRegister(Reg) && "Is a physreg")(static_cast <bool> (!Register::isPhysicalRegister(Reg)
 && "Is a physreg") ? void (0) : __assert_fail ("!Register::isPhysicalRegister(Reg) && \"Is a physreg\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10327
, __extension__ __PRETTY_FUNCTION__));

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
// If this is an InlineAsm we have to match the registers required, not the
// notional registers required by the type.

RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
                 std::nullopt); // This is not an ABI copy.
SDValue Chain = DAG.getEntryNode();

if (ExtendType == ISD::ANY_EXTEND) {
  auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
  if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
    ExtendType = PreferredExtendIt->second;
}
RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
PendingExports.push_back(Chain);
10344}

10346#include "llvm/CodeGen/SelectionDAGISel.h"

10348/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
10349/// entry block, return true.  This includes arguments used by switches, since
10350/// the switch may expand into multiple basic blocks.
10351static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
// With FastISel active, we may be splitting blocks, so force creation
// of virtual registers for all non-dead arguments.
if (FastISel)
  return A->use_empty();

const BasicBlock &Entry = A->getParent()->front();
for (const User *U : A->users())
  if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
    return false;  // Use not in entry block.

return true;
10363}

10365using ArgCopyElisionMapTy =
  DenseMap<const Argument *,
           std::pair<const AllocaInst *, const StoreInst *>>;

10369/// Scan the entry block of the function in FuncInfo for arguments that look
10370/// like copies into a local alloca. Record any copied arguments in
10371/// ArgCopyElisionCandidates.
10372static void
10373findArgumentCopyElisionCandidates(const DataLayout &DL,
                                FunctionLoweringInfo *FuncInfo,
                                ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
// Record the state of every static alloca used in the entry block. Argument
// allocas are all used in the entry block, so we need approximately as many
// entries as we have arguments.
enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
unsigned NumArgs = FuncInfo->Fn->arg_size();
StaticAllocas.reserve(NumArgs * 2);

auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
  if (!V)
    return nullptr;
  V = V->stripPointerCasts();
  const auto *AI = dyn_cast<AllocaInst>(V);
  if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
    return nullptr;
  auto Iter = StaticAllocas.insert({AI, Unknown});
  return &Iter.first->second;
};

// Look for stores of arguments to static allocas. Look through bitcasts and
// GEPs to handle type coercions, as long as the alloca is fully initialized
// by the store. Any non-store use of an alloca escapes it and any subsequent
// unanalyzed store might write it.
// FIXME: Handle structs initialized with multiple stores.
for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
  // Look for stores, and handle non-store uses conservatively.
  const auto *SI = dyn_cast<StoreInst>(&I);
  if (!SI) {
    // We will look through cast uses, so ignore them completely.
    if (I.isCast())
      continue;
    // Ignore debug info and pseudo op intrinsics, they don't escape or store
    // to allocas.
    if (I.isDebugOrPseudoInst())
      continue;
    // This is an unknown instruction. Assume it escapes or writes to all
    // static alloca operands.
    for (const Use &U : I.operands()) {
      if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
        *Info = StaticAllocaInfo::Clobbered;
    }
    continue;
  }

  // If the stored value is a static alloca, mark it as escaped.
  if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
    *Info = StaticAllocaInfo::Clobbered;

  // Check if the destination is a static alloca.
  const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
  StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
  if (!Info)
    continue;
  const AllocaInst *AI = cast<AllocaInst>(Dst);

  // Skip allocas that have been initialized or clobbered.
  if (*Info != StaticAllocaInfo::Unknown)
    continue;

  // Check if the stored value is an argument, and that this store fully
  // initializes the alloca.
  // If the argument type has padding bits we can't directly forward a pointer
  // as the upper bits may contain garbage.
  // Don't elide copies from the same argument twice.
  const Value *Val = SI->getValueOperand()->stripPointerCasts();
  const auto *Arg = dyn_cast<Argument>(Val);
  if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
      Arg->getType()->isEmptyTy() ||
      DL.getTypeStoreSize(Arg->getType()) !=
          DL.getTypeAllocSize(AI->getAllocatedType()) ||
      !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
      ArgCopyElisionCandidates.count(Arg)) {
    *Info = StaticAllocaInfo::Clobbered;
    continue;
  }

  LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Found argument copy elision candidate: "
 << *AI << '\n'; } } while (false)
                    << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Found argument copy elision candidate: "
 << *AI << '\n'; } } while (false);

  // Mark this alloca and store for argument copy elision.
  *Info = StaticAllocaInfo::Elidable;
  ArgCopyElisionCandidates.insert({Arg, {AI, SI}});

  // Stop scanning if we've seen all arguments. This will happen early in -O0
  // builds, which is useful, because -O0 builds have large entry blocks and
  // many allocas.
  if (ArgCopyElisionCandidates.size() == NumArgs)
    break;
}
10465}

10467/// Try to elide argument copies from memory into a local alloca. Succeeds if
10468/// ArgVal is a load from a suitable fixed stack object.
10469static void tryToElideArgumentCopy(
  FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains,
  DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
  SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
  ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
  SDValue ArgVal, bool &ArgHasUses) {
// Check if this is a load from a fixed stack object.
auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
if (!LNode)
  return;
auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
if (!FINode)
  return;

// Check that the fixed stack object is the right size and alignment.
// Look at the alignment that the user wrote on the alloca instead of looking
// at the stack object.
auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
assert(ArgCopyIter != ArgCopyElisionCandidates.end())(static_cast <bool> (ArgCopyIter != ArgCopyElisionCandidates
.end()) ? void (0) : __assert_fail ("ArgCopyIter != ArgCopyElisionCandidates.end()"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10487
, __extension__ __PRETTY_FUNCTION__));
const AllocaInst *AI = ArgCopyIter->second.first;
int FixedIndex = FINode->getIndex();
int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
int OldIndex = AllocaIndex;
MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed due to bad fixed stack "
 "object size\n"; } } while (false)
      dbgs() << "  argument copy elision failed due to bad fixed stack "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed due to bad fixed stack "
 "object size\n"; } } while (false)
                "object size\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed due to bad fixed stack "
 "object size\n"; } } while (false);
  return;
}
Align RequiredAlignment = AI->getAlign();
if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
  LLVM_DEBUG(dbgs() << "  argument copy elision failed: alignment of alloca "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed: alignment of alloca "
 "greater than stack argument alignment (" << DebugStr(
RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign
(FixedIndex)) << ")\n"; } } while (false)
                       "greater than stack argument alignment ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed: alignment of alloca "
 "greater than stack argument alignment (" << DebugStr(
RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign
(FixedIndex)) << ")\n"; } } while (false)
                    << DebugStr(RequiredAlignment) << " vs "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed: alignment of alloca "
 "greater than stack argument alignment (" << DebugStr(
RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign
(FixedIndex)) << ")\n"; } } while (false)
                    << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "  argument copy elision failed: alignment of alloca "
 "greater than stack argument alignment (" << DebugStr(
RequiredAlignment) << " vs " << DebugStr(MFI.getObjectAlign
(FixedIndex)) << ")\n"; } } while (false);
  return;
}

// Perform the elision. Delete the old stack object and replace its only use
// in the variable info map. Mark the stack object as mutable.
LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
 Arg << " to " << *AI << '\n' << "  Replacing frame index "
 << OldIndex << " with " << FixedIndex <<
 '\n'; }; } } while (false)
  dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
 Arg << " to " << *AI << '\n' << "  Replacing frame index "
 << OldIndex << " with " << FixedIndex <<
 '\n'; }; } } while (false)
         << "  Replacing frame index " << OldIndex << " with " << FixedIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
 Arg << " to " << *AI << '\n' << "  Replacing frame index "
 << OldIndex << " with " << FixedIndex <<
 '\n'; }; } } while (false)
         << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
 Arg << " to " << *AI << '\n' << "  Replacing frame index "
 << OldIndex << " with " << FixedIndex <<
 '\n'; }; } } while (false)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
 Arg << " to " << *AI << '\n' << "  Replacing frame index "
 << OldIndex << " with " << FixedIndex <<
 '\n'; }; } } while (false);
MFI.RemoveStackObject(OldIndex);
MFI.setIsImmutableObjectIndex(FixedIndex, false);
AllocaIndex = FixedIndex;
ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
Chains.push_back(ArgVal.getValue(1));

// Avoid emitting code for the store implementing the copy.
const StoreInst *SI = ArgCopyIter->second.second;
ElidedArgCopyInstrs.insert(SI);

// Check for uses of the argument again so that we can avoid exporting ArgVal
// if it is't used by anything other than the store.
for (const Value *U : Arg.users()) {
  if (U != SI) {
    ArgHasUses = true;
    break;
  }
}
10533}

10535void SelectionDAGISel::LowerArguments(const Function &F) {
SelectionDAG &DAG = SDB->DAG;
SDLoc dl = SDB->getCurSDLoc();
const DataLayout &DL = DAG.getDataLayout();
SmallVector<ISD::InputArg, 16> Ins;

// In Naked functions we aren't going to save any registers.
if (F.hasFnAttribute(Attribute::Naked))
  return;

if (!FuncInfo->CanLowerReturn) {
  // Put in an sret pointer parameter before all the other parameters.
  SmallVector<EVT, 1> ValueVTs;
  ComputeValueVTs(*TLI, DAG.getDataLayout(),
                  F.getReturnType()->getPointerTo(
                      DAG.getDataLayout().getAllocaAddrSpace()),
                  ValueVTs);

  // NOTE: Assuming that a pointer will never break down to more than one VT
  // or one register.
  ISD::ArgFlagsTy Flags;
  Flags.setSRet();
  MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
  ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
                       ISD::InputArg::NoArgIndex, 0);
  Ins.push_back(RetArg);
}

// Look for stores of arguments to static allocas. Mark such arguments with a
// flag to ask the target to give us the memory location of that argument if
// available.
ArgCopyElisionMapTy ArgCopyElisionCandidates;
findArgumentCopyElisionCandidates(DL, FuncInfo.get(),
                                  ArgCopyElisionCandidates);

// Set up the incoming argument description vector.
for (const Argument &Arg : F.args()) {
  unsigned ArgNo = Arg.getArgNo();
  SmallVector<EVT, 4> ValueVTs;
  ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
  bool isArgValueUsed = !Arg.use_empty();
  unsigned PartBase = 0;
  Type *FinalType = Arg.getType();
  if (Arg.hasAttribute(Attribute::ByVal))
    FinalType = Arg.getParamByValType();
  bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
      FinalType, F.getCallingConv(), F.isVarArg(), DL);
  for (unsigned Value = 0, NumValues = ValueVTs.size();
       Value != NumValues; ++Value) {
    EVT VT = ValueVTs[Value];
    Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
    ISD::ArgFlagsTy Flags;


    if (Arg.getType()->isPointerTy()) {
      Flags.setPointer();
      Flags.setPointerAddrSpace(
          cast<PointerType>(Arg.getType())->getAddressSpace());
    }
    if (Arg.hasAttribute(Attribute::ZExt))
      Flags.setZExt();
    if (Arg.hasAttribute(Attribute::SExt))
      Flags.setSExt();
    if (Arg.hasAttribute(Attribute::InReg)) {
      // If we are using vectorcall calling convention, a structure that is
      // passed InReg - is surely an HVA
      if (F.getCallingConv() == CallingConv::X86_VectorCall &&
          isa<StructType>(Arg.getType())) {
        // The first value of a structure is marked
        if (0 == Value)
          Flags.setHvaStart();
        Flags.setHva();
      }
      // Set InReg Flag
      Flags.setInReg();
    }
    if (Arg.hasAttribute(Attribute::StructRet))
      Flags.setSRet();
    if (Arg.hasAttribute(Attribute::SwiftSelf))
      Flags.setSwiftSelf();
    if (Arg.hasAttribute(Attribute::SwiftAsync))
      Flags.setSwiftAsync();
    if (Arg.hasAttribute(Attribute::SwiftError))
      Flags.setSwiftError();
    if (Arg.hasAttribute(Attribute::ByVal))
      Flags.setByVal();
    if (Arg.hasAttribute(Attribute::ByRef))
      Flags.setByRef();
    if (Arg.hasAttribute(Attribute::InAlloca)) {
      Flags.setInAlloca();
      // Set the byval flag for CCAssignFn callbacks that don't know about
      // inalloca.  This way we can know how many bytes we should've allocated
      // and how many bytes a callee cleanup function will pop.  If we port
      // inalloca to more targets, we'll have to add custom inalloca handling
      // in the various CC lowering callbacks.
      Flags.setByVal();
    }
    if (Arg.hasAttribute(Attribute::Preallocated)) {
      Flags.setPreallocated();
      // Set the byval flag for CCAssignFn callbacks that don't know about
      // preallocated.  This way we can know how many bytes we should've
      // allocated and how many bytes a callee cleanup function will pop.  If
      // we port preallocated to more targets, we'll have to add custom
      // preallocated handling in the various CC lowering callbacks.
      Flags.setByVal();
    }

    // Certain targets (such as MIPS), may have a different ABI alignment
    // for a type depending on the context. Give the target a chance to
    // specify the alignment it wants.
    const Align OriginalAlignment(
        TLI->getABIAlignmentForCallingConv(ArgTy, DL));
    Flags.setOrigAlign(OriginalAlignment);

    Align MemAlign;
    Type *ArgMemTy = nullptr;
    if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
        Flags.isByRef()) {
      if (!ArgMemTy)
        ArgMemTy = Arg.getPointeeInMemoryValueType();

      uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);

      // For in-memory arguments, size and alignment should be passed from FE.
      // BE will guess if this info is not there but there are cases it cannot
      // get right.
      if (auto ParamAlign = Arg.getParamStackAlign())
        MemAlign = *ParamAlign;
      else if ((ParamAlign = Arg.getParamAlign()))
        MemAlign = *ParamAlign;
      else
        MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL));
      if (Flags.isByRef())
        Flags.setByRefSize(MemSize);
      else
        Flags.setByValSize(MemSize);
    } else if (auto ParamAlign = Arg.getParamStackAlign()) {
      MemAlign = *ParamAlign;
    } else {
      MemAlign = OriginalAlignment;
    }
    Flags.setMemAlign(MemAlign);

    if (Arg.hasAttribute(Attribute::Nest))
      Flags.setNest();
    if (NeedsRegBlock)
      Flags.setInConsecutiveRegs();
    if (ArgCopyElisionCandidates.count(&Arg))
      Flags.setCopyElisionCandidate();
    if (Arg.hasAttribute(Attribute::Returned))
      Flags.setReturned();

    MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
        *CurDAG->getContext(), F.getCallingConv(), VT);
    unsigned NumRegs = TLI->getNumRegistersForCallingConv(
        *CurDAG->getContext(), F.getCallingConv(), VT);
    for (unsigned i = 0; i != NumRegs; ++i) {
      // For scalable vectors, use the minimum size; individual targets
      // are responsible for handling scalable vector arguments and
      // return values.
      ISD::InputArg MyFlags(
          Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
          PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
      if (NumRegs > 1 && i == 0)
        MyFlags.Flags.setSplit();
      // if it isn't first piece, alignment must be 1
      else if (i > 0) {
        MyFlags.Flags.setOrigAlign(Align(1));
        if (i == NumRegs - 1)
          MyFlags.Flags.setSplitEnd();
      }
      Ins.push_back(MyFlags);
    }
    if (NeedsRegBlock && Value == NumValues - 1)
      Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
    PartBase += VT.getStoreSize().getKnownMinValue();
  }
}

// Call the target to set up the argument values.
SmallVector<SDValue, 8> InVals;
SDValue NewRoot = TLI->LowerFormalArguments(
    DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);

// Verify that the target's LowerFormalArguments behaved as expected.
assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&(static_cast <bool> (NewRoot.getNode() && NewRoot
.getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!"
) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10721
, __extension__ __PRETTY_FUNCTION__))
       "LowerFormalArguments didn't return a valid chain!")(static_cast <bool> (NewRoot.getNode() && NewRoot
.getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!"
) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10721
, __extension__ __PRETTY_FUNCTION__));
assert(InVals.size() == Ins.size() &&(static_cast <bool> (InVals.size() == Ins.size() &&
 "LowerFormalArguments didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10723
, __extension__ __PRETTY_FUNCTION__))
       "LowerFormalArguments didn't emit the correct number of values!")(static_cast <bool> (InVals.size() == Ins.size() &&
 "LowerFormalArguments didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10723
, __extension__ __PRETTY_FUNCTION__));
LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
    assert(InVals[i].getNode() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
           "LowerFormalArguments emitted a null value!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
    assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
           "LowerFormalArguments emitted a value with the wrong type!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
  }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
 "LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
 ("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10727
, __extension__ __PRETTY_FUNCTION__)); (static_cast <bool>
 (EVT(Ins[i].VT) == InVals[i].getValueType() && "LowerFormalArguments emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10729
, __extension__ __PRETTY_FUNCTION__)); } }; } } while (false);

// Update the DAG with the new chain value resulting from argument lowering.
DAG.setRoot(NewRoot);

// Set up the argument values.
unsigned i = 0;
if (!FuncInfo->CanLowerReturn) {
  // Create a virtual register for the sret pointer, and put in a copy
  // from the sret argument into it.
  SmallVector<EVT, 1> ValueVTs;
  ComputeValueVTs(*TLI, DAG.getDataLayout(),
                  F.getReturnType()->getPointerTo(
                      DAG.getDataLayout().getAllocaAddrSpace()),
                  ValueVTs);
  MVT VT = ValueVTs[0].getSimpleVT();
  MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
  std::optional<ISD::NodeType> AssertOp;
  SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
                                      nullptr, F.getCallingConv(), AssertOp);

  MachineFunction& MF = SDB->DAG.getMachineFunction();
  MachineRegisterInfo& RegInfo = MF.getRegInfo();
  Register SRetReg =
      RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
  FuncInfo->DemoteRegister = SRetReg;
  NewRoot =
      SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
  DAG.setRoot(NewRoot);

  // i indexes lowered arguments.  Bump it past the hidden sret argument.
  ++i;
}

SmallVector<SDValue, 4> Chains;
DenseMap<int, int> ArgCopyElisionFrameIndexMap;
for (const Argument &Arg : F.args()) {
  SmallVector<SDValue, 4> ArgValues;
  SmallVector<EVT, 4> ValueVTs;
  ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
  unsigned NumValues = ValueVTs.size();
  if (NumValues == 0)
    continue;

  bool ArgHasUses = !Arg.use_empty();

  // Elide the copying store if the target loaded this argument from a
  // suitable fixed stack object.
  if (Ins[i].Flags.isCopyElisionCandidate()) {
    tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
                           ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
                           InVals[i], ArgHasUses);
  }

  // If this argument is unused then remember its value. It is used to generate
  // debugging information.
  bool isSwiftErrorArg =
      TLI->supportSwiftError() &&
      Arg.hasAttribute(Attribute::SwiftError);
  if (!ArgHasUses && !isSwiftErrorArg) {
    SDB->setUnusedArgValue(&Arg, InVals[i]);

    // Also remember any frame index for use in FastISel.
    if (FrameIndexSDNode *FI =
        dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
      FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
  }

  for (unsigned Val = 0; Val != NumValues; ++Val) {
    EVT VT = ValueVTs[Val];
    MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
                                                    F.getCallingConv(), VT);
    unsigned NumParts = TLI->getNumRegistersForCallingConv(
        *CurDAG->getContext(), F.getCallingConv(), VT);

    // Even an apparent 'unused' swifterror argument needs to be returned. So
    // we do generate a copy for it that can be used on return from the
    // function.
    if (ArgHasUses || isSwiftErrorArg) {
      std::optional<ISD::NodeType> AssertOp;
      if (Arg.hasAttribute(Attribute::SExt))
        AssertOp = ISD::AssertSext;
      else if (Arg.hasAttribute(Attribute::ZExt))
        AssertOp = ISD::AssertZext;

      ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
                                           PartVT, VT, nullptr,
                                           F.getCallingConv(), AssertOp));
    }

    i += NumParts;
  }

  // We don't need to do anything else for unused arguments.
  if (ArgValues.empty())
    continue;

  // Note down frame index.
  if (FrameIndexSDNode *FI =
      dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
    FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());

  SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
                                   SDB->getCurSDLoc());

  SDB->setValue(&Arg, Res);
  if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
    // We want to associate the argument with the frame index, among
    // involved operands, that correspond to the lowest address. The
    // getCopyFromParts function, called earlier, is swapping the order of
    // the operands to BUILD_PAIR depending on endianness. The result of
    // that swapping is that the least significant bits of the argument will
    // be in the first operand of the BUILD_PAIR node, and the most
    // significant bits will be in the second operand.
    unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
    if (LoadSDNode *LNode =
        dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
      if (FrameIndexSDNode *FI =
          dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
        FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
  }

  // Analyses past this point are naive and don't expect an assertion.
  if (Res.getOpcode() == ISD::AssertZext)
    Res = Res.getOperand(0);

  // Update the SwiftErrorVRegDefMap.
  if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
    unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
    if (Register::isVirtualRegister(Reg))
      SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(),
                                 Reg);
  }

  // If this argument is live outside of the entry block, insert a copy from
  // wherever we got it to the vreg that other BB's will reference it as.
  if (Res.getOpcode() == ISD::CopyFromReg) {
    // If we can, though, try to skip creating an unnecessary vreg.
    // FIXME: This isn't very clean... it would be nice to make this more
    // general.
    unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
    if (Register::isVirtualRegister(Reg)) {
      FuncInfo->ValueMap[&Arg] = Reg;
      continue;
    }
  }
  if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
    FuncInfo->InitializeRegForValue(&Arg);
    SDB->CopyToExportRegsIfNeeded(&Arg);
  }
}

if (!Chains.empty()) {
  Chains.push_back(NewRoot);
  NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
}

DAG.setRoot(NewRoot);

assert(i == InVals.size() && "Argument register count mismatch!")(static_cast <bool> (i == InVals.size() && "Argument register count mismatch!"
) ? void (0) : __assert_fail ("i == InVals.size() && \"Argument register count mismatch!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10890
, __extension__ __PRETTY_FUNCTION__));

// If any argument copy elisions occurred and we have debug info, update the
// stale frame indices used in the dbg.declare variable info table.
MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
  for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
    auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
    if (I != ArgCopyElisionFrameIndexMap.end())
      VI.Slot = I->second;
  }
}

// Finally, if the target has anything special to do, allow it to do so.
emitFunctionEntryCode();
10905}

10907/// Handle PHI nodes in successor blocks.  Emit code into the SelectionDAG to
10908/// ensure constants are generated when needed.  Remember the virtual registers
10909/// that need to be added to the Machine PHI nodes as input.  We cannot just
10910/// directly add them, because expansion might result in multiple MBB's for one
10911/// BB.  As such, the start of the BB might correspond to a different MBB than
10912/// the end.
10913void
10914SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;

// Check PHI nodes in successors that expect a value to be available from this
// block.
for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
  if (!isa<PHINode>(SuccBB->begin())) continue;
  MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];

  // If this terminator has multiple identical successors (common for
  // switches), only handle each succ once.
  if (!SuccsHandled.insert(SuccMBB).second)
    continue;

  MachineBasicBlock::iterator MBBI = SuccMBB->begin();

  // At this point we know that there is a 1-1 correspondence between LLVM PHI
  // nodes and Machine PHI nodes, but the incoming operands have not been
  // emitted yet.
  for (const PHINode &PN : SuccBB->phis()) {
    // Ignore dead phi's.
    if (PN.use_empty())
      continue;

    // Skip empty types
    if (PN.getType()->isEmptyTy())
      continue;

    unsigned Reg;
    const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);

    if (const auto *C = dyn_cast<Constant>(PHIOp)) {
      unsigned &RegOut = ConstantsOut[C];
      if (RegOut == 0) {
        RegOut = FuncInfo.CreateRegs(C);
        // We need to zero/sign extend ConstantInt phi operands to match
        // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
        ISD::NodeType ExtendType = ISD::ANY_EXTEND;
        if (auto *CI = dyn_cast<ConstantInt>(C))
          ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
                                                  : ISD::ZERO_EXTEND;
        CopyValueToVirtualRegister(C, RegOut, ExtendType);
      }
      Reg = RegOut;
    } else {
      DenseMap<const Value *, Register>::iterator I =
        FuncInfo.ValueMap.find(PHIOp);
      if (I != FuncInfo.ValueMap.end())
        Reg = I->second;
      else {
        assert(isa<AllocaInst>(PHIOp) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) &&
 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)
) && "Didn't codegen value into a register!??") ? void
 (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10968
, __extension__ __PRETTY_FUNCTION__))
               FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) &&
 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)
) && "Didn't codegen value into a register!??") ? void
 (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10968
, __extension__ __PRETTY_FUNCTION__))
               "Didn't codegen value into a register!??")(static_cast <bool> (isa<AllocaInst>(PHIOp) &&
 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)
) && "Didn't codegen value into a register!??") ? void
 (0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 10968
, __extension__ __PRETTY_FUNCTION__));
        Reg = FuncInfo.CreateRegs(PHIOp);
        CopyValueToVirtualRegister(PHIOp, Reg);
      }
    }

    // Remember that this register needs to added to the machine PHI node as
    // the input for this MBB.
    SmallVector<EVT, 4> ValueVTs;
    ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
    for (EVT VT : ValueVTs) {
      const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
      for (unsigned i = 0; i != NumRegisters; ++i)
        FuncInfo.PHINodesToUpdate.push_back(
            std::make_pair(&*MBBI++, Reg + i));
      Reg += NumRegisters;
    }
  }
}

ConstantsOut.clear();
10989}

10991MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
MachineFunction::iterator I(MBB);
if (++I == FuncInfo.MF->end())
  return nullptr;
return &*I;
10996}

10998/// During lowering new call nodes can be created (such as memset, etc.).
10999/// Those will become new roots of the current DAG, but complications arise
11000/// when they are tail calls. In such cases, the call lowering will update
11001/// the root, but the builder still needs to know that a tail call has been
11002/// lowered in order to avoid generating an additional return.
11003void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
// If the node is null, we do have a tail call.
if (MaybeTC.getNode() != nullptr)
  DAG.setRoot(MaybeTC);
else
  HasTailCall = true;
11009}

11011void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
                                      MachineBasicBlock *SwitchMBB,
                                      MachineBasicBlock *DefaultMBB) {
MachineFunction *CurMF = FuncInfo.MF;
MachineBasicBlock *NextMBB = nullptr;
MachineFunction::iterator BBI(W.MBB);
if (++BBI != FuncInfo.MF->end())
  NextMBB = &*BBI;

unsigned Size = W.LastCluster - W.FirstCluster + 1;

BranchProbabilityInfo *BPI = FuncInfo.BPI;

if (Size == 2 && W.MBB == SwitchMBB) {
  // If any two of the cases has the same destination, and if one value
  // is the same as the other, but has one bit unset that the other has set,
  // use bit manipulation to do two compares at once.  For example:
  // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
  // TODO: This could be extended to merge any 2 cases in switches with 3
  // cases.
  // TODO: Handle cases where W.CaseBB != SwitchBB.
  CaseCluster &Small = *W.FirstCluster;
  CaseCluster &Big = *W.LastCluster;

  if (Small.Low == Small.High && Big.Low == Big.High &&
      Small.MBB == Big.MBB) {
    const APInt &SmallValue = Small.Low->getValue();
    const APInt &BigValue = Big.Low->getValue();

    // Check that there is only one bit different.
    APInt CommonBit = BigValue ^ SmallValue;
    if (CommonBit.isPowerOf2()) {
      SDValue CondLHS = getValue(Cond);
      EVT VT = CondLHS.getValueType();
      SDLoc DL = getCurSDLoc();

      SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
                               DAG.getConstant(CommonBit, DL, VT));
      SDValue Cond = DAG.getSetCC(
          DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
          ISD::SETEQ);

      // Update successor info.
      // Both Small and Big will jump to Small.BB, so we sum up the
      // probabilities.
      addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
      if (BPI)
        addSuccessorWithProb(
            SwitchMBB, DefaultMBB,
            // The default destination is the first successor in IR.
            BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
      else
        addSuccessorWithProb(SwitchMBB, DefaultMBB);

      // Insert the true branch.
      SDValue BrCond =
          DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
                      DAG.getBasicBlock(Small.MBB));
      // Insert the false branch.
      BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
                           DAG.getBasicBlock(DefaultMBB));

      DAG.setRoot(BrCond);
      return;
    }
  }
}

if (TM.getOptLevel() != CodeGenOpt::None) {
  // Here, we order cases by probability so the most likely case will be
  // checked first. However, two clusters can have the same probability in
  // which case their relative ordering is non-deterministic. So we use Low
  // as a tie-breaker as clusters are guaranteed to never overlap.
  llvm::sort(W.FirstCluster, W.LastCluster + 1,
             [](const CaseCluster &a, const CaseCluster &b) {
    return a.Prob != b.Prob ?
           a.Prob > b.Prob :
           a.Low->getValue().slt(b.Low->getValue());
  });

  // Rearrange the case blocks so that the last one falls through if possible
  // without changing the order of probabilities.
  for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
    --I;
    if (I->Prob > W.LastCluster->Prob)
      break;
    if (I->Kind == CC_Range && I->MBB == NextMBB) {
      std::swap(*I, *W.LastCluster);
      break;
    }
  }
}

// Compute total probability.
BranchProbability DefaultProb = W.DefaultProb;
BranchProbability UnhandledProbs = DefaultProb;
for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
  UnhandledProbs += I->Prob;

MachineBasicBlock *CurMBB = W.MBB;
for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
  bool FallthroughUnreachable = false;
  MachineBasicBlock *Fallthrough;
  if (I == W.LastCluster) {
    // For the last cluster, fall through to the default destination.
    Fallthrough = DefaultMBB;
    FallthroughUnreachable = isa<UnreachableInst>(
        DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
  } else {
    Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
    CurMF->insert(BBI, Fallthrough);
    // Put Cond in a virtual register to make it available from the new blocks.
    ExportFromCurrentBlock(Cond);
  }
  UnhandledProbs -= I->Prob;

  switch (I->Kind) {
    case CC_JumpTable: {
      // FIXME: Optimize away range check based on pivot comparisons.
      JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
      SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;

      // The jump block hasn't been inserted yet; insert it here.
      MachineBasicBlock *JumpMBB = JT->MBB;
      CurMF->insert(BBI, JumpMBB);

      auto JumpProb = I->Prob;
      auto FallthroughProb = UnhandledProbs;

      // If the default statement is a target of the jump table, we evenly
      // distribute the default probability to successors of CurMBB. Also
      // update the probability on the edge from JumpMBB to Fallthrough.
      for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
                                            SE = JumpMBB->succ_end();
           SI != SE; ++SI) {
        if (*SI == DefaultMBB) {
          JumpProb += DefaultProb / 2;
          FallthroughProb -= DefaultProb / 2;
          JumpMBB->setSuccProbability(SI, DefaultProb / 2);
          JumpMBB->normalizeSuccProbs();
          break;
        }
      }

      if (FallthroughUnreachable)
        JTH->FallthroughUnreachable = true;

      if (!JTH->FallthroughUnreachable)
        addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
      addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
      CurMBB->normalizeSuccProbs();

      // The jump table header will be inserted in our current block, do the
      // range check, and fall through to our fallthrough block.
      JTH->HeaderBB = CurMBB;
      JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.

      // If we're in the right place, emit the jump table header right now.
      if (CurMBB == SwitchMBB) {
        visitJumpTableHeader(*JT, *JTH, SwitchMBB);
        JTH->Emitted = true;
      }
      break;
    }
    case CC_BitTests: {
      // FIXME: Optimize away range check based on pivot comparisons.
      BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];

      // The bit test blocks haven't been inserted yet; insert them here.
      for (BitTestCase &BTC : BTB->Cases)
        CurMF->insert(BBI, BTC.ThisBB);

      // Fill in fields of the BitTestBlock.
      BTB->Parent = CurMBB;
      BTB->Default = Fallthrough;

      BTB->DefaultProb = UnhandledProbs;
      // If the cases in bit test don't form a contiguous range, we evenly
      // distribute the probability on the edge to Fallthrough to two
      // successors of CurMBB.
      if (!BTB->ContiguousRange) {
        BTB->Prob += DefaultProb / 2;
        BTB->DefaultProb -= DefaultProb / 2;
      }

      if (FallthroughUnreachable)
        BTB->FallthroughUnreachable = true;

      // If we're in the right place, emit the bit test header right now.
      if (CurMBB == SwitchMBB) {
        visitBitTestHeader(*BTB, SwitchMBB);
        BTB->Emitted = true;
      }
      break;
    }
    case CC_Range: {
      const Value *RHS, *LHS, *MHS;
      ISD::CondCode CC;
      if (I->Low == I->High) {
        // Check Cond == I->Low.
        CC = ISD::SETEQ;
        LHS = Cond;
        RHS=I->Low;
        MHS = nullptr;
      } else {
        // Check I->Low <= Cond <= I->High.
        CC = ISD::SETLE;
        LHS = I->Low;
        MHS = Cond;
        RHS = I->High;
      }

      // If Fallthrough is unreachable, fold away the comparison.
      if (FallthroughUnreachable)
        CC = ISD::SETTRUE;

      // The false probability is the sum of all unhandled cases.
      CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
                   getCurSDLoc(), I->Prob, UnhandledProbs);

      if (CurMBB == SwitchMBB)
        visitSwitchCase(CB, SwitchMBB);
      else
        SL->SwitchCases.push_back(CB);

      break;
    }
  }
  CurMBB = Fallthrough;
}
11241}

11243unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
                                            CaseClusterIt First,
                                            CaseClusterIt Last) {
return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
  if (X.Prob != CC.Prob)
    return X.Prob > CC.Prob;

  // Ties are broken by comparing the case value.
  return X.Low->getValue().slt(CC.Low->getValue());
});
11253}

11255void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
                                      const SwitchWorkListItem &W,
                                      Value *Cond,
                                      MachineBasicBlock *SwitchMBB) {
assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&(static_cast <bool> (W.FirstCluster->Low->getValue
().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?"
) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11260
, __extension__ __PRETTY_FUNCTION__))
       "Clusters not sorted?")(static_cast <bool> (W.FirstCluster->Low->getValue
().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?"
) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11260
, __extension__ __PRETTY_FUNCTION__));

assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!")(static_cast <bool> (W.LastCluster - W.FirstCluster + 1
 >= 2 && "Too small to split!") ? void (0) : __assert_fail
 ("W.LastCluster - W.FirstCluster + 1 >= 2 && \"Too small to split!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11262
, __extension__ __PRETTY_FUNCTION__));

// Balance the tree based on branch probabilities to create a near-optimal (in
// terms of search time given key frequency) binary search tree. See e.g. Kurt
// Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
CaseClusterIt LastLeft = W.FirstCluster;
CaseClusterIt FirstRight = W.LastCluster;
auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
auto RightProb = FirstRight->Prob + W.DefaultProb / 2;

// Move LastLeft and FirstRight towards each other from opposite directions to
// find a partitioning of the clusters which balances the probability on both
// sides. If LeftProb and RightProb are equal, alternate which side is
// taken to ensure 0-probability nodes are distributed evenly.
unsigned I = 0;
while (LastLeft + 1 < FirstRight) {
  if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
    LeftProb += (++LastLeft)->Prob;
  else
    RightProb += (--FirstRight)->Prob;
  I++;
}

while (true) {
  // Our binary search tree differs from a typical BST in that ours can have up
  // to three values in each leaf. The pivot selection above doesn't take that
  // into account, which means the tree might require more nodes and be less
  // efficient. We compensate for this here.

  unsigned NumLeft = LastLeft - W.FirstCluster + 1;
  unsigned NumRight = W.LastCluster - FirstRight + 1;

  if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
    // If one side has less than 3 clusters, and the other has more than 3,
    // consider taking a cluster from the other side.

    if (NumLeft < NumRight) {
      // Consider moving the first cluster on the right to the left side.
      CaseCluster &CC = *FirstRight;
      unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
      unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
      if (LeftSideRank <= RightSideRank) {
        // Moving the cluster to the left does not demote it.
        ++LastLeft;
        ++FirstRight;
        continue;
      }
    } else {
      assert(NumRight < NumLeft)(static_cast <bool> (NumRight < NumLeft) ? void (0) :
 __assert_fail ("NumRight < NumLeft", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 11310, __extension__ __PRETTY_FUNCTION__));
      // Consider moving the last element on the left to the right side.
      CaseCluster &CC = *LastLeft;
      unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
      unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
      if (RightSideRank <= LeftSideRank) {
        // Moving the cluster to the right does not demot it.
        --LastLeft;
        --FirstRight;
        continue;
      }
    }
  }
  break;
}

assert(LastLeft + 1 == FirstRight)(static_cast <bool> (LastLeft + 1 == FirstRight) ? void
 (0) : __assert_fail ("LastLeft + 1 == FirstRight", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 11326, __extension__ __PRETTY_FUNCTION__));
assert(LastLeft >= W.FirstCluster)(static_cast <bool> (LastLeft >= W.FirstCluster) ? void
 (0) : __assert_fail ("LastLeft >= W.FirstCluster", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 11327, __extension__ __PRETTY_FUNCTION__));
assert(FirstRight <= W.LastCluster)(static_cast <bool> (FirstRight <= W.LastCluster) ? void
 (0) : __assert_fail ("FirstRight <= W.LastCluster", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 11328, __extension__ __PRETTY_FUNCTION__));

// Use the first element on the right as pivot since we will make less-than
// comparisons against it.
CaseClusterIt PivotCluster = FirstRight;
assert(PivotCluster > W.FirstCluster)(static_cast <bool> (PivotCluster > W.FirstCluster) ?
 void (0) : __assert_fail ("PivotCluster > W.FirstCluster"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11333
, __extension__ __PRETTY_FUNCTION__));
assert(PivotCluster <= W.LastCluster)(static_cast <bool> (PivotCluster <= W.LastCluster) ?
 void (0) : __assert_fail ("PivotCluster <= W.LastCluster"
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11334
, __extension__ __PRETTY_FUNCTION__));

CaseClusterIt FirstLeft = W.FirstCluster;
CaseClusterIt LastRight = W.LastCluster;

const ConstantInt *Pivot = PivotCluster->Low;

// New blocks will be inserted immediately after the current one.
MachineFunction::iterator BBI(W.MBB);
++BBI;

// We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
// we can branch to its destination directly if it's squeezed exactly in
// between the known lower bound and Pivot - 1.
MachineBasicBlock *LeftMBB;
if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
    FirstLeft->Low == W.GE &&
    (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
  LeftMBB = FirstLeft->MBB;
} else {
  LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
  FuncInfo.MF->insert(BBI, LeftMBB);
  WorkList.push_back(
      {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
  // Put Cond in a virtual register to make it available from the new blocks.
  ExportFromCurrentBlock(Cond);
}

// Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
// single cluster, RHS.Low == Pivot, and we can branch to its destination
// directly if RHS.High equals the current upper bound.
MachineBasicBlock *RightMBB;
if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
    W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
  RightMBB = FirstRight->MBB;
} else {
  RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
  FuncInfo.MF->insert(BBI, RightMBB);
  WorkList.push_back(
      {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
  // Put Cond in a virtual register to make it available from the new blocks.
  ExportFromCurrentBlock(Cond);
}

// Create the CaseBlock record that will be used to lower the branch.
CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
             getCurSDLoc(), LeftProb, RightProb);

if (W.MBB == SwitchMBB)
  visitSwitchCase(CB, SwitchMBB);
else
  SL->SwitchCases.push_back(CB);
11386}

11388// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
11389// from the swith statement.
11390static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
                                          BranchProbability PeeledCaseProb) {
if (PeeledCaseProb == BranchProbability::getOne())
  return BranchProbability::getZero();
BranchProbability SwitchProb = PeeledCaseProb.getCompl();

uint32_t Numerator = CaseProb.getNumerator();
uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
return BranchProbability(Numerator, std::max(Numerator, Denominator));
11399}

11401// Try to peel the top probability case if it exceeds the threshold.
11402// Return current MachineBasicBlock for the switch statement if the peeling
11403// does not occur.
11404// If the peeling is performed, return the newly created MachineBasicBlock
11405// for the peeled switch statement. Also update Clusters to remove the peeled
11406// case. PeeledCaseProb is the BranchProbability for the peeled case.
11407MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
  const SwitchInst &SI, CaseClusterVector &Clusters,
  BranchProbability &PeeledCaseProb) {
MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
// Don't perform if there is only one cluster or optimizing for size.
if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
    TM.getOptLevel() == CodeGenOpt::None ||
    SwitchMBB->getParent()->getFunction().hasMinSize())
  return SwitchMBB;

BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
unsigned PeeledCaseIndex = 0;
bool SwitchPeeled = false;
for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
  CaseCluster &CC = Clusters[Index];
  if (CC.Prob < TopCaseProb)
    continue;
  TopCaseProb = CC.Prob;
  PeeledCaseIndex = Index;
  SwitchPeeled = true;
}
if (!SwitchPeeled)
  return SwitchMBB;

LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: "
 << TopCaseProb << "\n"; } } while (false)
                  << TopCaseProb << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: "
 << TopCaseProb << "\n"; } } while (false);

// Record the MBB for the peeled switch statement.
MachineFunction::iterator BBI(SwitchMBB);
++BBI;
MachineBasicBlock *PeeledSwitchMBB =
    FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
FuncInfo.MF->insert(BBI, PeeledSwitchMBB);

ExportFromCurrentBlock(SI.getCondition());
auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
                        nullptr,   nullptr,      TopCaseProb.getCompl()};
lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);

Clusters.erase(PeeledCaseIt);
for (CaseCluster &CC : Clusters) {
  LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: "
 << CC.Prob << "\n"; } } while (false)
      dbgs() << "Scale the probablity for one cluster, before scaling: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: "
 << CC.Prob << "\n"; } } while (false)
             << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: "
 << CC.Prob << "\n"; } } while (false);
  CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
  LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "After scaling: " << CC.Prob
 << "\n"; } } while (false);
}
PeeledCaseProb = TopCaseProb;
return PeeledSwitchMBB;
11457}

11459void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
// Extract cases from the switch.
BranchProbabilityInfo *BPI = FuncInfo.BPI;
CaseClusterVector Clusters;
Clusters.reserve(SI.getNumCases());
for (auto I : SI.cases()) {
  MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
  const ConstantInt *CaseVal = I.getCaseValue();
  BranchProbability Prob =
      BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
          : BranchProbability(1, SI.getNumCases() + 1);
  Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
}

MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];

// Cluster adjacent cases with the same destination. We do this at all
// optimization levels because it's cheap to do and will make codegen faster
// if there are many clusters.
sortAndRangeify(Clusters);

// The branch probablity of the peeled case.
BranchProbability PeeledCaseProb = BranchProbability::getZero();
MachineBasicBlock *PeeledSwitchMBB =
    peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);

// If there is only the default destination, jump there directly.
MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
if (Clusters.empty()) {
  assert(PeeledSwitchMBB == SwitchMBB)(static_cast <bool> (PeeledSwitchMBB == SwitchMBB) ? void
 (0) : __assert_fail ("PeeledSwitchMBB == SwitchMBB", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 11488, __extension__ __PRETTY_FUNCTION__));
  SwitchMBB->addSuccessor(DefaultMBB);
  if (DefaultMBB != NextBlock(SwitchMBB)) {
    DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
                            getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
  }
  return;
}

SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI());
SL->findBitTestClusters(Clusters, &SI);

LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
  dbgs() << "Case clusters: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
  for (const CaseCluster &C : Clusters) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
    if (C.Kind == CC_JumpTable)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
      dbgs() << "JT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
    if (C.Kind == CC_BitTests)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
      dbgs() << "BT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)

    C.Low->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
    if (C.Low != C.High) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
      dbgs() << '-';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
      C.High->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
    }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
    dbgs() << ' ';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
  }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
  dbgs() << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
})do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
 &C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
 "JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
 dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
);

assert(!Clusters.empty())(static_cast <bool> (!Clusters.empty()) ? void (0) : __assert_fail
 ("!Clusters.empty()", "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 11518, __extension__ __PRETTY_FUNCTION__));
SwitchWorkList WorkList;
CaseClusterIt First = Clusters.begin();
CaseClusterIt Last = Clusters.end() - 1;
auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
// Scale the branchprobability for DefaultMBB if the peel occurs and
// DefaultMBB is not replaced.
if (PeeledCaseProb != BranchProbability::getZero() &&
    DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
  DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
WorkList.push_back(
    {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});

while (!WorkList.empty()) {
  SwitchWorkListItem W = WorkList.pop_back_val();
  unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;

  if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
      !DefaultMBB->getParent()->getFunction().hasMinSize()) {
    // For optimized builds, lower large range as a balanced binary tree.
    splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
    continue;
  }

  lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
}
11544}

11546void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
auto DL = getCurSDLoc();
EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
setValue(&I, DAG.getStepVector(DL, ResultVT));
11551}

11553void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

SDLoc DL = getCurSDLoc();
SDValue V = getValue(I.getOperand(0));
assert(VT == V.getValueType() && "Malformed vector.reverse!")(static_cast <bool> (VT == V.getValueType() && "Malformed vector.reverse!"
) ? void (0) : __assert_fail ("VT == V.getValueType() && \"Malformed vector.reverse!\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11559
, __extension__ __PRETTY_FUNCTION__));

if (VT.isScalableVector()) {
  setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V));
  return;
}

// Use VECTOR_SHUFFLE for the fixed-length vector
// to maintain existing behavior.
SmallVector<int, 8> Mask;
unsigned NumElts = VT.getVectorMinNumElements();
for (unsigned i = 0; i != NumElts; ++i)
  Mask.push_back(NumElts - 1 - i);

setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
11574}

11576void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
auto DL = getCurSDLoc();
SDValue InVec = getValue(I.getOperand(0));
EVT OutVT =
    InVec.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());

unsigned OutNumElts = OutVT.getVectorMinNumElements();

// ISD Node needs the input vectors split into two equal parts
SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
                         DAG.getVectorIdxConstant(0, DL));
SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
                         DAG.getVectorIdxConstant(OutNumElts, DL));

// Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
// legalisation and combines.
if (OutVT.isFixedLengthVector()) {
  SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
                                      createStrideMask(0, 2, OutNumElts));
  SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
                                     createStrideMask(1, 2, OutNumElts));
  SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
  setValue(&I, Res);
  return;
}

SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
                          DAG.getVTList(OutVT, OutVT), Lo, Hi);
setValue(&I, Res);
11605}

11607void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
auto DL = getCurSDLoc();
EVT InVT = getValue(I.getOperand(0)).getValueType();
SDValue InVec0 = getValue(I.getOperand(0));
SDValue InVec1 = getValue(I.getOperand(1));
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());

// Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
// legalisation and combines.
if (OutVT.isFixedLengthVector()) {
  unsigned NumElts = InVT.getVectorMinNumElements();
  SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
  setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
                                    createInterleaveMask(NumElts, 2)));
  return;
}

SDValue Res = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
                          DAG.getVTList(InVT, InVT), InVec0, InVec1);
Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
                  Res.getValue(1));
setValue(&I, Res);
11630}

11632void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
SmallVector<EVT, 4> ValueVTs;
ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
                ValueVTs);
unsigned NumValues = ValueVTs.size();
if (NumValues == 0) return;

SmallVector<SDValue, 4> Values(NumValues);
SDValue Op = getValue(I.getOperand(0));

for (unsigned i = 0; i != NumValues; ++i)
  Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
                          SDValue(Op.getNode(), Op.getResNo() + i));

setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                         DAG.getVTList(ValueVTs), Values));
11648}

11650void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());

SDLoc DL = getCurSDLoc();
SDValue V1 = getValue(I.getOperand(0));
SDValue V2 = getValue(I.getOperand(1));
int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();

// VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
if (VT.isScalableVector()) {
  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
  setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
                           DAG.getConstant(Imm, DL, IdxVT)));
  return;
}

unsigned NumElts = VT.getVectorNumElements();

uint64_t Idx = (NumElts + Imm) % NumElts;

// Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
SmallVector<int, 8> Mask;
for (unsigned i = 0; i < NumElts; ++i)
  Mask.push_back(Idx + i);
setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
11676}

11678// Consider the following MIR after SelectionDAG, which produces output in
11679// phyregs in the first case or virtregs in the second case.
11680//
11681// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
11682// %5:gr32 = COPY $ebx
11683// %6:gr32 = COPY $edx
11684// %1:gr32 = COPY %6:gr32
11685// %0:gr32 = COPY %5:gr32
11686//
11687// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
11688// %1:gr32 = COPY %6:gr32
11689// %0:gr32 = COPY %5:gr32
11690//
11691// Given %0, we'd like to return $ebx in the first case and %5 in the second.
11692// Given %1, we'd like to return $edx in the first case and %6 in the second.
11693//
11694// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
11695// to a single virtreg (such as %0). The remaining outputs monotonically
11696// increase in virtreg number from there. If a callbr has no outputs, then it
11697// should not have a corresponding callbr landingpad; in fact, the callbr
11698// landingpad would not even be able to refer to such a callbr.
11699static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
MachineInstr *MI = MRI.def_begin(Reg)->getParent();
// There is definitely at least one copy.
assert(MI->getOpcode() == TargetOpcode::COPY &&(static_cast <bool> (MI->getOpcode() == TargetOpcode
::COPY && "start of copy chain MUST be COPY") ? void (
0) : __assert_fail ("MI->getOpcode() == TargetOpcode::COPY && \"start of copy chain MUST be COPY\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11703
, __extension__ __PRETTY_FUNCTION__))
       "start of copy chain MUST be COPY")(static_cast <bool> (MI->getOpcode() == TargetOpcode
::COPY && "start of copy chain MUST be COPY") ? void (
0) : __assert_fail ("MI->getOpcode() == TargetOpcode::COPY && \"start of copy chain MUST be COPY\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11703
, __extension__ __PRETTY_FUNCTION__));
Reg = MI->getOperand(1).getReg();
MI = MRI.def_begin(Reg)->getParent();
// There may be an optional second copy.
if (MI->getOpcode() == TargetOpcode::COPY) {
  assert(Reg.isVirtual() && "expected COPY of virtual register")(static_cast <bool> (Reg.isVirtual() && "expected COPY of virtual register"
) ? void (0) : __assert_fail ("Reg.isVirtual() && \"expected COPY of virtual register\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11708
, __extension__ __PRETTY_FUNCTION__));
  Reg = MI->getOperand(1).getReg();
  assert(Reg.isPhysical() && "expected COPY of physical register")(static_cast <bool> (Reg.isPhysical() && "expected COPY of physical register"
) ? void (0) : __assert_fail ("Reg.isPhysical() && \"expected COPY of physical register\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11710
, __extension__ __PRETTY_FUNCTION__));
  MI = MRI.def_begin(Reg)->getParent();
}
// The start of the chain must be an INLINEASM_BR.
assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&(static_cast <bool> (MI->getOpcode() == TargetOpcode
::INLINEASM_BR && "end of copy chain MUST be INLINEASM_BR"
) ? void (0) : __assert_fail ("MI->getOpcode() == TargetOpcode::INLINEASM_BR && \"end of copy chain MUST be INLINEASM_BR\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11715
, __extension__ __PRETTY_FUNCTION__))
       "end of copy chain MUST be INLINEASM_BR")(static_cast <bool> (MI->getOpcode() == TargetOpcode
::INLINEASM_BR && "end of copy chain MUST be INLINEASM_BR"
) ? void (0) : __assert_fail ("MI->getOpcode() == TargetOpcode::INLINEASM_BR && \"end of copy chain MUST be INLINEASM_BR\""
, "llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp", 11715
, __extension__ __PRETTY_FUNCTION__));
return Reg;
11717}

11719// We must do this walk rather than the simpler
11720//   setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
11721// otherwise we will end up with copies of virtregs only valid along direct
11722// edges.
11723void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
SmallVector<EVT, 8> ResultVTs;
SmallVector<SDValue, 8> ResultValues;
const auto *CBR =
    cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());

const TargetLowering &TLI = DAG.getTargetLoweringInfo();
const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();

unsigned InitialDef = FuncInfo.ValueMap[CBR];
SDValue Chain = DAG.getRoot();

// Re-parse the asm constraints string.
TargetLowering::AsmOperandInfoVector TargetConstraints =
    TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
for (auto &T : TargetConstraints) {
  SDISelAsmOperandInfo OpInfo(T);
  if (OpInfo.Type != InlineAsm::isOutput)
    continue;

  // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
  // individual constraint.
  TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);

  switch (OpInfo.ConstraintType) {
  case TargetLowering::C_Register:
  case TargetLowering::C_RegisterClass: {
    // Fill in OpInfo.AssignedRegs.Regs.
    getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);

    // getRegistersForValue may produce 1 to many registers based on whether
    // the OpInfo.ConstraintVT is legal on the target or not.
    for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) {
      Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
      if (Register::isPhysicalRegister(OriginalDef))
        FuncInfo.MBB->addLiveIn(OriginalDef);
      // Update the assigned registers to use the original defs.
      OpInfo.AssignedRegs.Regs[i] = OriginalDef;
    }

    SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
        DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
    ResultValues.push_back(V);
    ResultVTs.push_back(OpInfo.ConstraintVT);
    break;
  }
  case TargetLowering::C_Other: {
    SDValue Flag;
    SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
                                                OpInfo, DAG);
    ++InitialDef;
    ResultValues.push_back(V);
    ResultVTs.push_back(OpInfo.ConstraintVT);
    break;
  }
  default:
    break;
  }
}
SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
                        DAG.getVTList(ResultVTs), ResultValues);
setValue(&I, V);
11786}

←

/build/source/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

→

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/MachineValueType.h"
34#include "llvm/CodeGen/Register.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DebugLoc.h"
38#include "llvm/IR/Instruction.h"
39#include "llvm/IR/Instructions.h"
40#include "llvm/IR/Metadata.h"
41#include "llvm/IR/Operator.h"
42#include "llvm/Support/AlignOf.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/ErrorHandling.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56#include <utility>

58namespace llvm {

60class APInt;
61class Constant;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;

72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG.  Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
82};

84namespace ISD {

/// Node predicates

88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);

109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);

113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);

117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

121/// Returns true if the specified node is a vector where all elements can
122/// be truncated to the specified element size without a loss in meaning.
123bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed);

125/// Return true if the node has at least one operand and all operands of the
126/// specified node are ISD::UNDEF.
127bool allOperandsUndef(const SDNode *N);

129/// Return true if the specified node is FREEZE(UNDEF).
130bool isFreezeUndef(const SDNode *N);

132} // end namespace ISD

134//===----------------------------------------------------------------------===//
135/// Unlike LLVM values, Selection DAG nodes may return multiple
136/// values as the result of a computation.  Many nodes return multiple values,
137/// from loads (which define a token and a return value) to ADDC (which returns
138/// a result and a carry value), to calls (which may return an arbitrary number
139/// of values).
140///
141/// As such, each use of a SelectionDAG computation must indicate the node that
142/// computes it as well as which return value to use from that node.  This pair
143/// of information is represented with the SDValue value type.
144///
145class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
19
←
Null pointer value stored to 'Val.Node'→
unsigned ResNo = 0;     // Which return value of the node we are using.

151public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
237};

239template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
260};

262/// Allow casting operators to work directly on
263/// SDValues as if they were SDNode*'s.
264template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
}
270};
271template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
277};

279/// Represents a use of a SDNode. This class holds an SDValue,
280/// which records the SDNode being used and the result number, a
281/// pointer to the SDNode using the value, and Next and Prev pointers,
282/// which link together all the uses of an SDNode.
283///
284class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

294public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }
const SDNode *getUser() const { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

335private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
364};

366/// simplify_type specializations - Allow casting operators to work directly on
367/// SDValues as if they were SDNode*'s.
368template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
374};

376/// These are IR-level optimization flags that may be propagated to SDNodes.
377/// TODO: This data structure should be shared by the IR optimizer and the
378/// the backend.
379struct SDNodeFlags {
380private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

399public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
459};

461/// Represents one node in the SelectionDAG.
462///
463class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
464private:
/// The operation that this node performs.
int32_t NodeType;

468public:
/// Unique and persistent id per SDNode in the DAG. Used for debug printing.
/// We do not place that under `#if LLVM_ENABLE_ABI_BREAKING_CHECKS`
/// intentionally because it adds unneeded complexity without noticeable
/// benefits (see discussion with @thakis in D120714).
uint16_t PersistentId = 0xffff;

475protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with PersistentId.

480#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
481// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
482// and give the `pack` pragma push semantics.
483#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
484#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
485#else
486#define BEGIN_TWO_BYTE_PACK()
487#define END_TWO_BYTE_PACK()
488#endif

490BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class VPBaseLoadStoreSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;
  friend class VPGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   VPGatherScatterSDNode => enum ISD::MemIndexType
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class VPLoadSDNode;
  friend class VPStridedLoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;
  friend class VPGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class VPStoreSDNode;
  friend class VPStridedStoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;
  friend class VPScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
582END_TWO_BYTE_PACK()
583#undef BEGIN_TWO_BYTE_PACK
584#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

596private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

uint32_t CFIType = 0;

635public:
//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
687#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
689#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node is a vector predication operation.
bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 705, __extension__
 __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using iterator_category = std::forward_iterator_tag;
  using value_type = SDUse;
  using difference_type = std::ptrdiff_t;
  using pointer = value_type *;
  using reference = value_type &;

  use_iterator() = default;
  use_iterator(const use_iterator &I) = default;
  use_iterator &operator=(const use_iterator &) = default;

  bool operator==(const use_iterator &x) const { return Op == x.Op; }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 773, __extension__
 __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 784, __extension__
 __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 794, __extension__
 __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 922, __extension__
 __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return ArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

void setCFIType(uint32_t Type) { CFIType = Type; }
uint32_t getCFIType() const { return CFIType; }

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 987, __extension__
 __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1074protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
 "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1088, __extension__
 __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1090, __extension__
 __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1090, __extension__
 __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1095};

1097/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1098/// into SDNode creation functions.
1099/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1100/// from the original Instruction, and IROrder is the ordinal position of
1101/// the instruction.
1102/// When an SDNode is created after the DAG is being built, both DebugLoc and
1103/// the IROrder are propagated from the original SDNode.
1104/// So SDLoc class provides two constructors besides the default one, one to
1105/// be used by the DAGBuilder, the other to be used by others.
1106class SDLoc {
1107private:
DebugLoc DL;
int IROrder = 0;

1111public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1116, __extension__
 __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1123};

1125// Define inline functions from the SDValue class.

1127inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1133, __extension__
 __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1133, __extension__
 __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1134, __extension__
 __PRETTY_FUNCTION__));
1135}

1137inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
1139}

1141inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
32
←
Called C++ object pointer is null
1143}

1145inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1147}

1149inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1151}

1153inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1155}

1157inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1159}

1161inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1163}

1165inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1167}

1169inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1171}

1173inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1175}

1177inline bool SDValue::isUndef() const {
return Node->isUndef();
1179}

1181inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1183}

1185inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
1187}

1189inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1191}

1193inline void SDValue::dump() const {
return Node->dump();
1195}

1197inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1199}

1201inline void SDValue::dumpr() const {
return Node->dumpr();
1203}

1205inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1207}

1209// Define inline functions from the SDUse class.

1211inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode())
  V->addUse(*this);
1216}

1218inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V->addUse(*this);
1221}

1223inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1227}

1229/// This class is used to form a handle around another node that
1230/// is persistent and is updated across invocations of replaceAllUsesWith on its
1231/// operand.  This node should be directly created by end-users and not added to
1232/// the AllNodes list.
1233class HandleSDNode : public SDNode {
SDUse Op;

1236public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1255};

1257class AddrSpaceCastSDNode : public SDNode {
1258private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1262public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1272};

1274/// This is an abstract virtual class for memory operations.
1275class MemSDNode : public SDNode {
1276private:
// VT of in-memory value.
EVT MemoryVT;

1280protected:
/// Memory reference information.
MachineMemOperand *MMO;

1284public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getSuccessOrdering() const {
  return MMO->getSuccessOrdering();
}

/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation.  (For operations
/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::VP_STORE:
  case ISD::MSTORE:
  case ISD::VP_SCATTER:
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  switch (N->getOpcode()) {
  case ISD::LOAD:
  case ISD::STORE:
  case ISD::PREFETCH:
  case ISD::ATOMIC_CMP_SWAP:
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  case ISD::ATOMIC_SWAP:
  case ISD::ATOMIC_LOAD_ADD:
  case ISD::ATOMIC_LOAD_SUB:
  case ISD::ATOMIC_LOAD_AND:
  case ISD::ATOMIC_LOAD_CLR:
  case ISD::ATOMIC_LOAD_OR:
  case ISD::ATOMIC_LOAD_XOR:
  case ISD::ATOMIC_LOAD_NAND:
  case ISD::ATOMIC_LOAD_MIN:
  case ISD::ATOMIC_LOAD_MAX:
  case ISD::ATOMIC_LOAD_UMIN:
  case ISD::ATOMIC_LOAD_UMAX:
  case ISD::ATOMIC_LOAD_FADD:
  case ISD::ATOMIC_LOAD_FSUB:
  case ISD::ATOMIC_LOAD_FMAX:
  case ISD::ATOMIC_LOAD_FMIN:
  case ISD::ATOMIC_LOAD_UINC_WRAP:
  case ISD::ATOMIC_LOAD_UDEC_WRAP:
  case ISD::ATOMIC_LOAD:
  case ISD::ATOMIC_STORE:
  case ISD::MLOAD:
  case ISD::MSTORE:
  case ISD::MGATHER:
  case ISD::MSCATTER:
  case ISD::VP_LOAD:
  case ISD::VP_STORE:
  case ISD::VP_GATHER:
  case ISD::VP_SCATTER:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return true;
  default:
    return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
  }
}
1441};

1443/// This is an SDNode representing atomic operations.
1444class AtomicSDNode : public MemSDNode {
1445public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1450, __extension__
 __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1450, __extension__
 __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1467, __extension__
 __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UINC_WRAP ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UDEC_WRAP ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1496};

1498/// This SDNode is used for target intrinsics that touch
1499/// memory and need an associated MachineMemOperand. Its opcode may be
1500/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1501/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1502class MemIntrinsicSDNode : public MemSDNode {
1503public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1518};

1520/// This SDNode is used to implement the code generator
1521/// support for the llvm IR shufflevector instruction.  It combines elements
1522/// from two input vectors into a new input vector, with the selection and
1523/// ordering of elements determined by an array of integers, referred to as
1524/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1525/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1526/// An index of -1 is treated as undef, such that the code generator may put
1527/// any value in the corresponding element of the result.
1528class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1533protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1539public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return ArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
 ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1546, __extension__
 __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1553, __extension__
 __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1584};

1586class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1598public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isZero() const { return Value->isZero(); }
LLVM_DEPRECATED("use isZero instead", "isZero")__attribute__((deprecated("use isZero instead", "isZero")))
bool isNullValue() const { return isZero(); }
bool isAllOnes() const { return Value->isMinusOne(); }
LLVM_DEPRECATED("use isAllOnes instead", "isAllOnes")__attribute__((deprecated("use isAllOnes instead", "isAllOnes"
)))
bool isAllOnesValue() const { return isAllOnes(); }
bool isMaxSignedValue() const { return Value->isMaxValue(true); }
bool isMinSignedValue() const { return Value->isMinValue(true); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1625};

1627uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1629}

1631const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1633}

1635class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1645public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1680};

1682/// Returns true if \p V is a constant integer zero.
1683bool isNullConstant(SDValue V);

1685/// Returns true if \p V is an FP constant with a value of positive zero.
1686bool isNullFPConstant(SDValue V);

1688/// Returns true if \p V is an integer constant with all bits set.
1689bool isAllOnesConstant(SDValue V);

1691/// Returns true if \p V is a constant integer one.
1692bool isOneConstant(SDValue V);

1694/// Returns true if \p V is a constant min signed integer value.
1695bool isMinSignedConstant(SDValue V);

1697/// Returns true if \p V is a neutral element of Opc with Flags.
1698/// When OperandNo is 0, it checks that V is a left identity. Otherwise, it
1699/// checks that V is a right identity.
1700bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V,
                     unsigned OperandNo);

1703/// Return the non-bitcasted source operand of \p V if it exists.
1704/// If \p V is not a bitcasted value, it is returned as-is.
1705SDValue peekThroughBitcasts(SDValue V);

1707/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1708/// If \p V is not a bitcasted one-use value, it is returned as-is.
1709SDValue peekThroughOneUseBitcasts(SDValue V);

1711/// Return the non-extracted vector source operand of \p V if it exists.
1712/// If \p V is not an extracted subvector, it is returned as-is.
1713SDValue peekThroughExtractSubvectors(SDValue V);

1715/// Return the non-truncated source operand of \p V if it exists.
1716/// If \p V is not a truncation, it is returned as-is.
1717SDValue peekThroughTruncates(SDValue V);

1719/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1720/// constant is canonicalized to be operand 1.
1721bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1723/// If \p V is a bitwise not, returns the inverted operand. Otherwise returns
1724/// an empty SDValue. Only bits set in \p Mask are required to be inverted,
1725/// other bits may be arbitrary.
1726SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs);

1728/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1729ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1732/// Returns the SDNode if it is a demanded constant splat BuildVector or
1733/// constant int.
1734ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1738/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1739ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1741/// Returns the SDNode if it is a demanded constant splat BuildVector or
1742/// constant float.
1743ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1746/// Return true if the value is a constant 0 integer or a splatted vector of
1747/// a constant 0 integer (with no undefs by default).
1748/// Build vector implicit truncation is not an issue for null values.
1749bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1751/// Return true if the value is a constant 1 integer or a splatted vector of a
1752/// constant 1 integer (with no undefs).
1753/// Build vector implicit truncation is allowed, but the truncated bits need to
1754/// be zero.
1755bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1757/// Return true if the value is a constant -1 integer or a splatted vector of a
1758/// constant -1 integer (with no undefs).
1759/// Does not permit build vector implicit truncation.
1760bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1762/// Return true if \p V is either a integer or FP constant.
1763inline bool isIntOrFPConstant(SDValue V) {
return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1765}

1767class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1778public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1791};

1793class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1803public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1810};

1812/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1813/// the offet and size that are started/ended in the underlying FrameIndex.
1814class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1822public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1829, __extension__
 __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1833, __extension__
 __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1842};

1844/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1845/// the index of the basic block being probed. A pseudo probe serves as a place
1846/// holder and will be removed at the end of compilation. It does not have any
1847/// operand because we do not want the instruction selection to deal with any.
1848class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1859public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1868};

1870class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1881public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1889};

1891class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1907, __extension__
 __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1916, __extension__
 __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1921public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1927, __extension__
 __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 1932, __extension__
 __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1951};

1953/// Completely target-dependent object reference.
1954class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1961public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1973};

1975class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1987public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1993};

1995/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1996class BuildVectorSDNode : public SDNode {
1997public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

/// Extract the raw bit data from a build vector of Undef, Constant or
/// ConstantFP node elements. Each raw bit element will be \p
/// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
/// undefined elements are flagged in \p UndefElements.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                        SmallVectorImpl<APInt> &RawBitElements,
                        BitVector &UndefElements) const;

bool isConstant() const;

/// If this BuildVector is constant and represents the numerical series
/// "<a, a+n, a+2n, a+3n, ...>" where a is integer and n is a non-zero integer,
/// the value "<a,n>" is returned.
std::optional<std::pair<APInt, APInt>> isConstantSequence() const;

/// Recast bit data \p SrcBitElements to \p DstEltSizeInBits wide elements.
/// Undef elements are treated as zero, and entirely undefined elements are
/// flagged in \p DstUndefElements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                          SmallVectorImpl<APInt> &DstBitElements,
                          ArrayRef<APInt> SrcBitElements,
                          BitVector &DstUndefElements,
                          const BitVector &SrcUndefElements);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2128};

2130/// An SDNode that holds an arbitrary LLVM IR Value. This is
2131/// used when the SelectionDAG needs to make a simple reference to something
2132/// in the LLVM IR representation.
2133///
2134class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2143public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2150};

2152class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2161public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2167};

2169class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2177public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2183};

2185class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2195public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2201};

2203class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2215public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2224};

2226class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
 "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2233, __extension__
 __PRETTY_FUNCTION__));
}

2236public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2243};

2245class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2256public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2264};

2266class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2274public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2280};

2282class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2291public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2297};

2299/// This class is used to represent EVT's, which are used
2300/// to parameterize some operations.
2301class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2310public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2316};

2318/// Base class for LoadSDNode and StoreSDNode
2319class LSBaseSDNode : public MemSDNode {
2320public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2326, __extension__
 __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2349};

2351/// This class is used to represent ISD::LOAD nodes.
2352class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2360, __extension__
 __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2361, __extension__
 __PRETTY_FUNCTION__));
}

2364public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2377};

2379/// This class is used to represent ISD::STORE nodes.
2380class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2388, __extension__
 __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2389, __extension__
 __PRETTY_FUNCTION__));
}

2392public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2408};

2410/// This base class is used to represent VP_LOAD, VP_STORE,
2411/// EXPERIMENTAL_VP_STRIDED_LOAD and EXPERIMENTAL_VP_STRIDED_STORE nodes
2412class VPBaseLoadStoreSDNode : public MemSDNode {
2413public:
friend class SelectionDAG;

VPBaseLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &DL, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, DL, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2422, __extension__
 __PRETTY_FUNCTION__));
}

// VPStridedStoreSDNode (Chain, Data, Ptr,    Offset, Stride, Mask, EVL)
// VPStoreSDNode        (Chain, Data, Ptr,    Offset, Mask,   EVL)
// VPStridedLoadSDNode  (Chain, Ptr,  Offset, Stride, Mask,   EVL)
// VPLoadSDNode         (Chain, Ptr,  Offset, Mask,   EVL)
// Mask is a vector of i1 elements;
// the type of EVL is TLI.getVPExplicitVectorLengthTy().
const SDValue &getOffset() const {
  return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
                     getOpcode() == ISD::VP_LOAD)
                        ? 2
                        : 3);
}
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
                     getOpcode() == ISD::VP_LOAD)
                        ? 1
                        : 2);
}
const SDValue &getMask() const {
  switch (getOpcode()) {
  default:
    llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2446);
  case ISD::VP_LOAD:
    return getOperand(3);
  case ISD::VP_STORE:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return getOperand(4);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(5);
  }
}
const SDValue &getVectorLength() const {
  switch (getOpcode()) {
  default:
    llvm_unreachable("Invalid opcode")::llvm::llvm_unreachable_internal("Invalid opcode", "llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2459);
  case ISD::VP_LOAD:
    return getOperand(4);
  case ISD::VP_STORE:
  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
    return getOperand(5);
  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
    return getOperand(6);
  }
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD ||
         N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE ||
         N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
}
2487};

2489/// This class is used to represent a VP_LOAD node
2490class VPLoadSDNode : public VPBaseLoadStoreSDNode {
2491public:
friend class SelectionDAG;

VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
             ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
             EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = isExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getVectorLength() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2515};

2517/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
2518class VPStridedLoadSDNode : public VPBaseLoadStoreSDNode {
2519public:
friend class SelectionDAG;

VPStridedLoadSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
                    ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                    bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_LOAD, Order, DL, VTs,
                            AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getStride() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2545};

2547/// This class is used to represent a VP_STORE node
2548class VPStoreSDNode : public VPBaseLoadStoreSDNode {
2549public:
friend class SelectionDAG;

VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
              ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
              EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_STORE;
}
2580};

2582/// This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
2583class VPStridedStoreSDNode : public VPBaseLoadStoreSDNode {
2584public:
friend class SelectionDAG;

VPStridedStoreSDNode(unsigned Order, const DebugLoc &DL, SDVTList VTs,
                     ISD::MemIndexedMode AM, bool IsTrunc, bool IsCompressing,
                     EVT MemVT, MachineMemOperand *MMO)
    : VPBaseLoadStoreSDNode(ISD::EXPERIMENTAL_VP_STRIDED_STORE, Order, DL,
                            VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
  StoreSDNodeBits.IsCompressing = IsCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getStride() const { return getOperand(4); }
const SDValue &getMask() const { return getOperand(5); }
const SDValue &getVectorLength() const { return getOperand(6); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE;
}
2617};

2619/// This base class is used to represent MLOAD and MSTORE nodes
2620class MaskedLoadStoreSDNode : public MemSDNode {
2621public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2630, __extension__
 __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2659};

2661/// This class is used to represent an MLOAD node
2662class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2663public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2688};

2690/// This class is used to represent an MSTORE node
2691class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2692public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2722};

2724/// This is a base class used to represent
2725/// VP_GATHER and VP_SCATTER nodes
2726///
2727class VPGatherScatterSDNode : public MemSDNode {
2728public:
friend class SelectionDAG;

VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                      MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2736, __extension__
 __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
bool isIndexScaled() const {
  return !cast<ConstantSDNode>(getScale())->isOne();
}
bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); }

// In the both nodes address is Op1, mask is Op2:
// VPGatherSDNode  (Chain, base, index, scale, mask, vlen)
// VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
}
const SDValue &getIndex() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
}
const SDValue &getScale() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
}
const SDValue &getMask() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
}
const SDValue &getVectorLength() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER ||
         N->getOpcode() == ISD::VP_SCATTER;
}
2772};

2774/// This class is used to represent an VP_GATHER node
2775///
2776class VPGatherSDNode : public VPGatherScatterSDNode {
2777public:
friend class SelectionDAG;

VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
               MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER;
}
2788};

2790/// This class is used to represent an VP_SCATTER node
2791///
2792class VPScatterSDNode : public VPGatherScatterSDNode {
2793public:
friend class SelectionDAG;

VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_SCATTER;
}
2806};

2808/// This is a base class used to represent
2809/// MGATHER and MSCATTER nodes
2810///
2811class MaskedGatherScatterSDNode : public MemSDNode {
2812public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 2820, __extension__
 __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
bool isIndexScaled() const {
  return !cast<ConstantSDNode>(getScale())->isOne();
}
bool isIndexSigned() const { return isIndexTypeSigned(getIndexType()); }

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2845};

2847/// This class is used to represent an MGATHER node
2848///
2849class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2850public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2870};

2872/// This class is used to represent an MSCATTER node
2873///
2874class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2875public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2896};

2898/// An SDNode that represents everything that will be needed
2899/// to construct a MachineInstr. These nodes are created during the
2900/// instruction selection proper phase.
2901///
2902/// Note that the only supported way to set the `memoperands` is by calling the
2903/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2904/// inside the DAG rather than in the node.
2905class MachineSDNode : public SDNode {
2906private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2934public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return ArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return ArrayRef(cast<MachineMemOperand **>(MemRefs), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2960};

2962/// An SDNode that records if a register contains a value that is guaranteed to
2963/// be aligned accordingly.
2964class AssertAlignSDNode : public SDNode {
Align Alignment;

2967public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2976};

2978class SDNodeIterator {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2984public:
using iterator_category = std::forward_iterator_tag;
using value_type = SDNode;
using difference_type = std::ptrdiff_t;
using pointer = value_type *;
using reference = value_type &;

bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 3010, __extension__
 __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "llvm/include/llvm/CodeGen/SelectionDAGNodes.h", 3010, __extension__
 __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
3021};

3023template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
3036};

3038/// A representation of the largest SDNode, for use in sizeof().
3039///
3040/// This needs to be a union because the largest node differs on 32 bit systems
3041/// with 4 and 8 byte pointer alignment, respectively.
3042using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

3047/// The SDNode class with the greatest alignment requirement.
3048using MostAlignedSDNode = GlobalAddressSDNode;

3050namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

3128} // end namespace ISD

3130} // end namespace llvm

3132#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H

←

/build/source/llvm/include/llvm/CodeGen/SelectionDAG.h

1//===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SelectionDAG class, and transitively defines the
10// SDNode class and subclasses.
11//
12//===----------------------------------------------------------------------===//

14#ifndef LLVM_CODEGEN_SELECTIONDAG_H
15#define LLVM_CODEGEN_SELECTIONDAG_H

17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/ilist.h"
26#include "llvm/ADT/iterator.h"
27#include "llvm/ADT/iterator_range.h"
28#include "llvm/CodeGen/DAGCombine.h"
29#include "llvm/CodeGen/ISDOpcodes.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineMemOperand.h"
32#include "llvm/CodeGen/MachineValueType.h"
33#include "llvm/CodeGen/SelectionDAGNodes.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/Metadata.h"
37#include "llvm/Support/Allocator.h"
38#include "llvm/Support/ArrayRecycler.h"
39#include "llvm/Support/CodeGen.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/RecyclingAllocator.h"
42#include <cassert>
43#include <cstdint>
44#include <functional>
45#include <map>
46#include <string>
47#include <tuple>
48#include <utility>
49#include <vector>

51namespace llvm {

53class DIExpression;
54class DILabel;
55class DIVariable;
56class Function;
57class Pass;
58class Type;
59template <class GraphType> struct GraphTraits;
60template <typename T, unsigned int N> class SmallSetVector;
61template <typename T, typename Enable> struct FoldingSetTrait;
62class AAResults;
63class BlockAddress;
64class BlockFrequencyInfo;
65class Constant;
66class ConstantFP;
67class ConstantInt;
68class DataLayout;
69struct fltSemantics;
70class FunctionLoweringInfo;
71class FunctionVarLocs;
72class GlobalValue;
73struct KnownBits;
74class LLVMContext;
75class MachineBasicBlock;
76class MachineConstantPoolValue;
77class MCSymbol;
78class OptimizationRemarkEmitter;
79class ProfileSummaryInfo;
80class SDDbgValue;
81class SDDbgOperand;
82class SDDbgLabel;
83class SelectionDAG;
84class SelectionDAGTargetInfo;
85class TargetLibraryInfo;
86class TargetLowering;
87class TargetMachine;
88class TargetSubtargetInfo;
89class Value;

91template <typename T> class GenericSSAContext;
92using SSAContext = GenericSSAContext<Function>;
93template <typename T> class GenericUniformityInfo;
94using UniformityInfo = GenericUniformityInfo<SSAContext>;

96class SDVTListNode : public FoldingSetNode {
friend struct FoldingSetTrait<SDVTListNode>;

/// A reference to an Interned FoldingSetNodeID for this node.
/// The Allocator in SelectionDAG holds the data.
/// SDVTList contains all types which are frequently accessed in SelectionDAG.
/// The size of this list is not expected to be big so it won't introduce
/// a memory penalty.
FoldingSetNodeIDRef FastID;
const EVT *VTs;
unsigned int NumVTs;
/// The hash value for SDVTList is fixed, so cache it to avoid
/// hash calculation.
unsigned HashValue;

111public:
SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) :
    FastID(ID), VTs(VT), NumVTs(Num) {
  HashValue = ID.ComputeHash();
}

SDVTList getSDVTList() {
  SDVTList result = {VTs, NumVTs};
  return result;
}
121};

123/// Specialize FoldingSetTrait for SDVTListNode
124/// to avoid computing temp FoldingSetNodeID and hash value.
125template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> {
static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) {
  ID = X.FastID;
}

static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID,
                   unsigned IDHash, FoldingSetNodeID &TempID) {
  if (X.HashValue != IDHash)
    return false;
  return ID == X.FastID;
}

static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) {
  return X.HashValue;
}
140};

142template <> struct ilist_alloc_traits<SDNode> {
static void deleteNode(SDNode *) {
  llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!")::llvm::llvm_unreachable_internal("ilist_traits<SDNode> shouldn't see a deleteNode call!"
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 144);
}
146};

148/// Keeps track of dbg_value information through SDISel.  We do
149/// not build SDNodes for these so as not to perturb the generated code;
150/// instead the info is kept off to the side in this structure. Each SDNode may
151/// have one or more associated dbg_value entries. This information is kept in
152/// DbgValMap.
153/// Byval parameters are handled separately because they don't use alloca's,
154/// which busts the normal mechanism.  There is good reason for handling all
155/// parameters separately:  they may not have code generated for them, they
156/// should always go at the beginning of the function regardless of other code
157/// motion, and debug info for them is potentially useful even if the parameter
158/// is unused.  Right now only byval parameters are handled separately.
159class SDDbgInfo {
BumpPtrAllocator Alloc;
SmallVector<SDDbgValue*, 32> DbgValues;
SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
SmallVector<SDDbgLabel*, 4> DbgLabels;
using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>;
DbgValMapType DbgValMap;

167public:
SDDbgInfo() = default;
SDDbgInfo(const SDDbgInfo &) = delete;
SDDbgInfo &operator=(const SDDbgInfo &) = delete;

void add(SDDbgValue *V, bool isParameter);

void add(SDDbgLabel *L) { DbgLabels.push_back(L); }

/// Invalidate all DbgValues attached to the node and remove
/// it from the Node-to-DbgValues map.
void erase(const SDNode *Node);

void clear() {
  DbgValMap.clear();
  DbgValues.clear();
  ByvalParmDbgValues.clear();
  DbgLabels.clear();
  Alloc.Reset();
}

BumpPtrAllocator &getAlloc() { return Alloc; }

bool empty() const {
  return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty();
}

ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const {
  auto I = DbgValMap.find(Node);
  if (I != DbgValMap.end())
    return I->second;
  return ArrayRef<SDDbgValue*>();
}

using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator;
using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator;

DbgIterator DbgBegin() { return DbgValues.begin(); }
DbgIterator DbgEnd()   { return DbgValues.end(); }
DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); }
DbgIterator ByvalParmDbgEnd()   { return ByvalParmDbgValues.end(); }
DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); }
DbgLabelIterator DbgLabelEnd()   { return DbgLabels.end(); }
210};

212void checkForCycles(const SelectionDAG *DAG, bool force = false);

214/// This is used to represent a portion of an LLVM function in a low-level
215/// Data Dependence DAG representation suitable for instruction selection.
216/// This DAG is constructed as the first step of instruction selection in order
217/// to allow implementation of machine specific optimizations
218/// and code simplifications.
219///
220/// The representation used by the SelectionDAG is a target-independent
221/// representation, which has some similarities to the GCC RTL representation,
222/// but is significantly more simple, powerful, and is a graph form instead of a
223/// linear form.
224///
225class SelectionDAG {
const TargetMachine &TM;
const SelectionDAGTargetInfo *TSI = nullptr;
const TargetLowering *TLI = nullptr;
const TargetLibraryInfo *LibInfo = nullptr;
const FunctionVarLocs *FnVarLocs = nullptr;
MachineFunction *MF;
Pass *SDAGISelPass = nullptr;
LLVMContext *Context;
CodeGenOpt::Level OptLevel;

UniformityInfo *UA = nullptr;
FunctionLoweringInfo * FLI = nullptr;

/// The function-level optimization remark emitter.  Used to emit remarks
/// whenever manipulating the DAG.
OptimizationRemarkEmitter *ORE;

ProfileSummaryInfo *PSI = nullptr;
BlockFrequencyInfo *BFI = nullptr;

/// List of non-single value types.
FoldingSet<SDVTListNode> VTListMap;

/// Pool allocation for misc. objects that are created once per SelectionDAG.
BumpPtrAllocator Allocator;

/// The starting token.
SDNode EntryNode;

/// The root of the entire DAG.
SDValue Root;

/// A linked list of nodes in the current DAG.
ilist<SDNode> AllNodes;

/// The AllocatorType for allocating SDNodes. We use
/// pool allocation with recycling.
using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode,
                                             sizeof(LargestSDNode),
                                             alignof(MostAlignedSDNode)>;

/// Pool allocation for nodes.
NodeAllocatorType NodeAllocator;

/// This structure is used to memoize nodes, automatically performing
/// CSE with existing nodes when a duplicate is requested.
FoldingSet<SDNode> CSEMap;

/// Pool allocation for machine-opcode SDNode operands.
BumpPtrAllocator OperandAllocator;
ArrayRecycler<SDUse> OperandRecycler;

/// Tracks dbg_value and dbg_label information through SDISel.
SDDbgInfo *DbgInfo;

using CallSiteInfo = MachineFunction::CallSiteInfo;
using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl;

struct NodeExtraInfo {
  CallSiteInfo CSInfo;
  MDNode *HeapAllocSite = nullptr;
  MDNode *PCSections = nullptr;
  bool NoMerge = false;
};
/// Out-of-line extra information for SDNodes.
DenseMap<const SDNode *, NodeExtraInfo> SDEI;

/// PersistentId counter to be used when inserting the next
/// SDNode to this SelectionDAG. We do not place that under
/// `#if LLVM_ENABLE_ABI_BREAKING_CHECKS` intentionally because
/// it adds unneeded complexity without noticeable
/// benefits (see discussion with @thakis in D120714).
uint16_t NextPersistentId = 0;

300public:
/// Clients of various APIs that cause global effects on
/// the DAG can optionally implement this interface.  This allows the clients
/// to handle the various sorts of updates that happen.
///
/// A DAGUpdateListener automatically registers itself with DAG when it is
/// constructed, and removes itself when destroyed in RAII fashion.
struct DAGUpdateListener {
  DAGUpdateListener *const Next;
  SelectionDAG &DAG;

  explicit DAGUpdateListener(SelectionDAG &D)
    : Next(D.UpdateListeners), DAG(D) {
    DAG.UpdateListeners = this;
  }

  virtual ~DAGUpdateListener() {
    assert(DAG.UpdateListeners == this &&(static_cast <bool> (DAG.UpdateListeners == this &&
 "DAGUpdateListeners must be destroyed in LIFO order") ? void
 (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 318, __extension__
 __PRETTY_FUNCTION__))
           "DAGUpdateListeners must be destroyed in LIFO order")(static_cast <bool> (DAG.UpdateListeners == this &&
 "DAGUpdateListeners must be destroyed in LIFO order") ? void
 (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 318, __extension__
 __PRETTY_FUNCTION__));
    DAG.UpdateListeners = Next;
  }

  /// The node N that was deleted and, if E is not null, an
  /// equivalent node E that replaced it.
  virtual void NodeDeleted(SDNode *N, SDNode *E);

  /// The node N that was updated.
  virtual void NodeUpdated(SDNode *N);

  /// The node N that was inserted.
  virtual void NodeInserted(SDNode *N);
};

struct DAGNodeDeletedListener : public DAGUpdateListener {
  std::function<void(SDNode *, SDNode *)> Callback;

  DAGNodeDeletedListener(SelectionDAG &DAG,
                         std::function<void(SDNode *, SDNode *)> Callback)
      : DAGUpdateListener(DAG), Callback(std::move(Callback)) {}

  void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); }

 private:
  virtual void anchor();
};

struct DAGNodeInsertedListener : public DAGUpdateListener {
  std::function<void(SDNode *)> Callback;

  DAGNodeInsertedListener(SelectionDAG &DAG,
                          std::function<void(SDNode *)> Callback)
      : DAGUpdateListener(DAG), Callback(std::move(Callback)) {}

  void NodeInserted(SDNode *N) override { Callback(N); }

private:
  virtual void anchor();
};

/// Help to insert SDNodeFlags automatically in transforming. Use
/// RAII to save and resume flags in current scope.
class FlagInserter {
  SelectionDAG &DAG;
  SDNodeFlags Flags;
  FlagInserter *LastInserter;

public:
  FlagInserter(SelectionDAG &SDAG, SDNodeFlags Flags)
      : DAG(SDAG), Flags(Flags),
        LastInserter(SDAG.getFlagInserter()) {
    SDAG.setFlagInserter(this);
  }
  FlagInserter(SelectionDAG &SDAG, SDNode *N)
      : FlagInserter(SDAG, N->getFlags()) {}

  FlagInserter(const FlagInserter &) = delete;
  FlagInserter &operator=(const FlagInserter &) = delete;
  ~FlagInserter() { DAG.setFlagInserter(LastInserter); }

  SDNodeFlags getFlags() const { return Flags; }
};

/// When true, additional steps are taken to
/// ensure that getConstant() and similar functions return DAG nodes that
/// have legal types. This is important after type legalization since
/// any illegally typed nodes generated after this point will not experience
/// type legalization.
bool NewNodesMustHaveLegalTypes = false;

389private:
/// DAGUpdateListener is a friend so it can manipulate the listener stack.
friend struct DAGUpdateListener;

/// Linked list of registered DAGUpdateListener instances.
/// This stack is maintained by DAGUpdateListener RAII.
DAGUpdateListener *UpdateListeners = nullptr;

/// Implementation of setSubgraphColor.
/// Return whether we had to truncate the search.
bool setSubgraphColorHelper(SDNode *N, const char *Color,
                            DenseSet<SDNode *> &visited,
                            int level, bool &printed);

template <typename SDNodeT, typename... ArgTypes>
SDNodeT *newSDNode(ArgTypes &&... Args) {
  return new (NodeAllocator.template Allocate<SDNodeT>())
      SDNodeT(std::forward<ArgTypes>(Args)...);
}

/// Build a synthetic SDNodeT with the given args and extract its subclass
/// data as an integer (e.g. for use in a folding set).
///
/// The args to this function are the same as the args to SDNodeT's
/// constructor, except the second arg (assumed to be a const DebugLoc&) is
/// omitted.
template <typename SDNodeT, typename... ArgTypes>
static uint16_t getSyntheticNodeSubclassData(unsigned IROrder,
                                             ArgTypes &&... Args) {
  // The compiler can reduce this expression to a constant iff we pass an
  // empty DebugLoc.  Thankfully, the debug location doesn't have any bearing
  // on the subclass data.
  return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...)
      .getRawSubclassData();
}

template <typename SDNodeTy>
static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order,
                                              SDVTList VTs, EVT MemoryVT,
                                              MachineMemOperand *MMO) {
  return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO)
       .getRawSubclassData();
}

void createOperands(SDNode *Node, ArrayRef<SDValue> Vals);

void removeOperands(SDNode *Node) {
  if (!Node->OperandList)
    return;
  OperandRecycler.deallocate(
      ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
      Node->OperandList);
  Node->NumOperands = 0;
  Node->OperandList = nullptr;
}
void CreateTopologicalOrder(std::vector<SDNode*>& Order);

446public:
// Maximum depth for recursive analysis such as computeKnownBits, etc.
static constexpr unsigned MaxRecursionDepth = 6;

explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level);
SelectionDAG(const SelectionDAG &) = delete;
SelectionDAG &operator=(const SelectionDAG &) = delete;
~SelectionDAG();

/// Prepare this SelectionDAG to process code in the given MachineFunction.
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
          Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
          UniformityInfo *UA, ProfileSummaryInfo *PSIin,
          BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs);

void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) {
  FLI = FuncInfo;
}

/// Clear state and free memory necessary to make this
/// SelectionDAG ready to process a new block.
void clear();

MachineFunction &getMachineFunction() const { return *MF; }
const Pass *getPass() const { return SDAGISelPass; }

const DataLayout &getDataLayout() const { return MF->getDataLayout(); }
const TargetMachine &getTarget() const { return TM; }
const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); }
template <typename STC> const STC &getSubtarget() const {
  return MF->getSubtarget<STC>();
}
const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
const TargetLibraryInfo &getLibInfo() const { return *LibInfo; }
const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; }
const UniformityInfo *getUniformityInfo() const { return UA; }
/// Returns the result of the AssignmentTrackingAnalysis pass if it's
/// available, otherwise return nullptr.
const FunctionVarLocs *getFunctionVarLocs() const { return FnVarLocs; }
LLVMContext *getContext() const { return Context; }
OptimizationRemarkEmitter &getORE() const { return *ORE; }
ProfileSummaryInfo *getPSI() const { return PSI; }
BlockFrequencyInfo *getBFI() const { return BFI; }

FlagInserter *getFlagInserter() { return Inserter; }
void setFlagInserter(FlagInserter *FI) { Inserter = FI; }

/// Just dump dot graph to a user-provided path and title.
/// This doesn't open the dot viewer program and
/// helps visualization when outside debugging session.
/// FileName expects absolute path. If provided
/// without any path separators then the file
/// will be created in the current directory.
/// Error will be emitted if the path is insane.
500#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void dumpDotGraph(const Twine &FileName, const Twine &Title);
502#endif

/// Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
void viewGraph(const std::string &Title);
void viewGraph();

508#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
std::map<const SDNode *, std::string> NodeGraphAttrs;
510#endif

/// Clear all previously defined node graph attributes.
/// Intended to be used from a debugging tool (eg. gdb).
void clearGraphAttrs();

/// Set graph attributes for a node. (eg. "color=red".)
void setGraphAttrs(const SDNode *N, const char *Attrs);

/// Get graph attributes for a node. (eg. "color=red".)
/// Used from getNodeAttributes.
std::string getGraphAttrs(const SDNode *N) const;

/// Convenience for setting node color attribute.
void setGraphColor(const SDNode *N, const char *Color);

/// Convenience for setting subgraph color attribute.
void setSubgraphColor(SDNode *N, const char *Color);

using allnodes_const_iterator = ilist<SDNode>::const_iterator;

allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); }
allnodes_const_iterator allnodes_end() const { return AllNodes.end(); }

using allnodes_iterator = ilist<SDNode>::iterator;

allnodes_iterator allnodes_begin() { return AllNodes.begin(); }
allnodes_iterator allnodes_end() { return AllNodes.end(); }

ilist<SDNode>::size_type allnodes_size() const {
  return AllNodes.size();
}

iterator_range<allnodes_iterator> allnodes() {
  return make_range(allnodes_begin(), allnodes_end());
}
iterator_range<allnodes_const_iterator> allnodes() const {
  return make_range(allnodes_begin(), allnodes_end());
}

/// Return the root tag of the SelectionDAG.
const SDValue &getRoot() const { return Root; }

/// Return the token chain corresponding to the entry of the function.
SDValue getEntryNode() const {
  return SDValue(const_cast<SDNode *>(&EntryNode), 0);
}

/// Set the current root tag of the SelectionDAG.
///
const SDValue &setRoot(SDValue N) {
  assert((!N.getNode() || N.getValueType() == MVT::Other) &&(static_cast <bool> ((!N.getNode() || N.getValueType() ==
 MVT::Other) && "DAG root value is not a chain!") ? void
 (0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 562, __extension__
 __PRETTY_FUNCTION__))
         "DAG root value is not a chain!")(static_cast <bool> ((!N.getNode() || N.getValueType() ==
 MVT::Other) && "DAG root value is not a chain!") ? void
 (0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 562, __extension__
 __PRETTY_FUNCTION__));
  if (N.getNode())
    checkForCycles(N.getNode(), this);
  Root = N;
  if (N.getNode())
    checkForCycles(this);
  return Root;
}

571#ifndef NDEBUG
void VerifyDAGDivergence();
573#endif

/// This iterates over the nodes in the SelectionDAG, folding
/// certain types of nodes together, or eliminating superfluous nodes.  The
/// Level argument controls whether Combine is allowed to produce nodes and
/// types that are illegal on the target.
void Combine(CombineLevel Level, AAResults *AA,
             CodeGenOpt::Level OptLevel);

/// This transforms the SelectionDAG into a SelectionDAG that
/// only uses types natively supported by the target.
/// Returns "true" if it made any changes.
///
/// Note that this is an involved process that may invalidate pointers into
/// the graph.
bool LegalizeTypes();

/// This transforms the SelectionDAG into a SelectionDAG that is
/// compatible with the target instruction selector, as indicated by the
/// TargetLowering object.
///
/// Note that this is an involved process that may invalidate pointers into
/// the graph.
void Legalize();

/// Transforms a SelectionDAG node and any operands to it into a node
/// that is compatible with the target instruction selector, as indicated by
/// the TargetLowering object.
///
/// \returns true if \c N is a valid, legal node after calling this.
///
/// This essentially runs a single recursive walk of the \c Legalize process
/// over the given node (and its operands). This can be used to incrementally
/// legalize the DAG. All of the nodes which are directly replaced,
/// potentially including N, are added to the output parameter \c
/// UpdatedNodes so that the delta to the DAG can be understood by the
/// caller.
///
/// When this returns false, N has been legalized in a way that make the
/// pointer passed in no longer valid. It may have even been deleted from the
/// DAG, and so it shouldn't be used further. When this returns true, the
/// N passed in is a legal node, and can be immediately processed as such.
/// This may still have done some work on the DAG, and will still populate
/// UpdatedNodes with any new nodes replacing those originally in the DAG.
bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes);

/// This transforms the SelectionDAG into a SelectionDAG
/// that only uses vector math operations supported by the target.  This is
/// necessary as a separate step from Legalize because unrolling a vector
/// operation can introduce illegal types, which requires running
/// LegalizeTypes again.
///
/// This returns true if it made any changes; in that case, LegalizeTypes
/// is called again before Legalize.
///
/// Note that this is an involved process that may invalidate pointers into
/// the graph.
bool LegalizeVectors();

/// This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNodes();

/// Remove the specified node from the system.  This node must
/// have no referrers.
void DeleteNode(SDNode *N);

/// Return an SDVTList that represents the list of values specified.
SDVTList getVTList(EVT VT);
SDVTList getVTList(EVT VT1, EVT VT2);
SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4);
SDVTList getVTList(ArrayRef<EVT> VTs);

//===--------------------------------------------------------------------===//
// Node creation methods.

/// Create a ConstantSDNode wrapping a constant value.
/// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
///
/// If only legal types can be produced, this does the necessary
/// transformations (e.g., if the vector element type is illegal).
/// @{
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
                    bool isTarget = false, bool isOpaque = false);
SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
                    bool isTarget = false, bool isOpaque = false);

SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
                           bool IsOpaque = false) {
  return getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT,
                     IsTarget, IsOpaque);
}

SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
                    bool isTarget = false, bool isOpaque = false);
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL,
                          bool isTarget = false);
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL,
                               bool LegalTypes = true);
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
                             bool isTarget = false);

SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT,
                          bool isOpaque = false) {
  return getConstant(Val, DL, VT, true, isOpaque);
}
SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT,
                          bool isOpaque = false) {
  return getConstant(Val, DL, VT, true, isOpaque);
}
SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
                          bool isOpaque = false) {
  return getConstant(Val, DL, VT, true, isOpaque);
}

/// Create a true or false constant of type \p VT using the target's
/// BooleanContent for type \p OpVT.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT);
/// @}

/// Create a ConstantFPSDNode wrapping a constant value.
/// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
///
/// If only legal types can be produced, this does the necessary
/// transformations (e.g., if the vector element type is illegal).
/// The forms that take a double should only be used for simple constants
/// that can be exactly represented in VT.  No checks are made.
/// @{
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT,
                      bool isTarget = false);
SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT,
                      bool isTarget = false);
SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT,
                      bool isTarget = false);
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) {
  return getConstantFP(Val, DL, VT, true);
}
SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) {
  return getConstantFP(Val, DL, VT, true);
}
SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) {
  return getConstantFP(Val, DL, VT, true);
}
/// @}

SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
                         int64_t offset = 0, bool isTargetGA = false,
                         unsigned TargetFlags = 0);
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
                               int64_t offset = 0, unsigned TargetFlags = 0) {
  return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags);
}
SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false);
SDValue getTargetFrameIndex(int FI, EVT VT) {
  return getFrameIndex(FI, VT, true);
}
SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false,
                     unsigned TargetFlags = 0);
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) {
  return getJumpTable(JTI, VT, true, TargetFlags);
}
SDValue getConstantPool(const Constant *C, EVT VT,
                        MaybeAlign Align = std::nullopt, int Offs = 0,
                        bool isT = false, unsigned TargetFlags = 0);
SDValue getTargetConstantPool(const Constant *C, EVT VT,
                              MaybeAlign Align = std::nullopt, int Offset = 0,
                              unsigned TargetFlags = 0) {
  return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
}
SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT,
                        MaybeAlign Align = std::nullopt, int Offs = 0,
                        bool isT = false, unsigned TargetFlags = 0);
SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT,
                              MaybeAlign Align = std::nullopt, int Offset = 0,
                              unsigned TargetFlags = 0) {
  return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
}
SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
                       unsigned TargetFlags = 0);
// When generating a branch to a BB, we don't in general know enough
// to provide debug info for the BB at that time, so keep this one around.
SDValue getBasicBlock(MachineBasicBlock *MBB);
SDValue getExternalSymbol(const char *Sym, EVT VT);
SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
                                unsigned TargetFlags = 0);
SDValue getMCSymbol(MCSymbol *Sym, EVT VT);

SDValue getValueType(EVT);
SDValue getRegister(unsigned Reg, EVT VT);
SDValue getRegisterMask(const uint32_t *RegMask);
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
                     MCSymbol *Label);
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0,
                        bool isTarget = false, unsigned TargetFlags = 0);
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT,
                              int64_t Offset = 0, unsigned TargetFlags = 0) {
  return getBlockAddress(BA, VT, Offset, true, TargetFlags);
}

SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg,
                     SDValue N) {
  return getNode(ISD::CopyToReg, dl, MVT::Other, Chain,
                 getRegister(Reg, N.getValueType()), N);
}

// This version of the getCopyToReg method takes an extra operand, which
// indicates that there is potentially an incoming glue value (if Glue is not
// null) and that there should be a glue result.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
                     SDValue Glue) {
  SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
  return getNode(ISD::CopyToReg, dl, VTs,
                 ArrayRef(Ops, Glue.getNode() ? 4 : 3));
}

// Similar to last getCopyToReg() except parameter Reg is a SDValue
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
                     SDValue Glue) {
  SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, Reg, N, Glue };
  return getNode(ISD::CopyToReg, dl, VTs,
                 ArrayRef(Ops, Glue.getNode() ? 4 : 3));
}

SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) {
  SDVTList VTs = getVTList(VT, MVT::Other);
  SDValue Ops[] = { Chain, getRegister(Reg, VT) };
  return getNode(ISD::CopyFromReg, dl, VTs, Ops);
}

// This version of the getCopyFromReg method takes an extra operand, which
// indicates that there is potentially an incoming glue value (if Glue is not
// null) and that there should be a glue result.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT,
                       SDValue Glue) {
  SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
  return getNode(ISD::CopyFromReg, dl, VTs,
                 ArrayRef(Ops, Glue.getNode() ? 3 : 2));
}

SDValue getCondCode(ISD::CondCode Cond);

/// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT,
/// which must be a vector type, must match the number of mask elements
/// NumElts. An integer mask element equal to -1 is treated as undefined.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
                         ArrayRef<int> Mask);

/// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
/// which must be a vector type, must match the number of operands in Ops.
/// The operands must have the same type as (or, for integers, a type wider
/// than) VT's element type.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) {
  // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
  return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
}

/// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
/// which must be a vector type, must match the number of operands in Ops.
/// The operands must have the same type as (or, for integers, a type wider
/// than) VT's element type.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) {
  // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
  return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
}

/// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all
/// elements. VT must be a vector type. Op's type must be the same as (or,
/// for integers, a type wider than) VT's element type.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) {
  // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
  if (Op.getOpcode() == ISD::UNDEF) {
    assert((VT.getVectorElementType() == Op.getValueType() ||(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 852, __extension__
 __PRETTY_FUNCTION__))
            (VT.isInteger() &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 852, __extension__
 __PRETTY_FUNCTION__))
             VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 852, __extension__
 __PRETTY_FUNCTION__))
           "A splatted value must have a width equal or (for integers) "(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 852, __extension__
 __PRETTY_FUNCTION__))
           "greater than the vector element type!")(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 852, __extension__
 __PRETTY_FUNCTION__));
    return getNode(ISD::UNDEF, SDLoc(), VT);
  }

  SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op);
  return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
}

// Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all
// elements.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) {
  if (Op.getOpcode() == ISD::UNDEF) {
    assert((VT.getVectorElementType() == Op.getValueType() ||(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 868, __extension__
 __PRETTY_FUNCTION__))
            (VT.isInteger() &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 868, __extension__
 __PRETTY_FUNCTION__))
             VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 868, __extension__
 __PRETTY_FUNCTION__))
           "A splatted value must have a width equal or (for integers) "(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 868, __extension__
 __PRETTY_FUNCTION__))
           "greater than the vector element type!")(static_cast <bool> ((VT.getVectorElementType() == Op.getValueType
() || (VT.isInteger() && VT.getVectorElementType().bitsLE
(Op.getValueType()))) && "A splatted value must have a width equal or (for integers) "
 "greater than the vector element type!") ? void (0) : __assert_fail
 ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 868, __extension__
 __PRETTY_FUNCTION__));
    return getNode(ISD::UNDEF, SDLoc(), VT);
  }
  return getNode(ISD::SPLAT_VECTOR, DL, VT, Op);
}

/// Returns a node representing a splat of one value into all lanes
/// of the provided vector type.  This is a utility which returns
/// either a BUILD_VECTOR or SPLAT_VECTOR depending on the
/// scalability of the desired vector type.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op) {
  assert(VT.isVector() && "Can't splat to non-vector type")(static_cast <bool> (VT.isVector() && "Can't splat to non-vector type"
) ? void (0) : __assert_fail ("VT.isVector() && \"Can't splat to non-vector type\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 879, __extension__
 __PRETTY_FUNCTION__));
  return VT.isScalableVector() ?
    getSplatVector(VT, DL, Op) : getSplatBuildVector(VT, DL, Op);
}

/// Returns a vector of type ResVT whose elements contain the linear sequence
///   <0, Step, Step * 2, Step * 3, ...>
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal);

/// Returns a vector of type ResVT whose elements contain the linear sequence
///   <0, 1, 2, 3, ...>
SDValue getStepVector(const SDLoc &DL, EVT ResVT);

/// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
/// the shuffle node in input but with swapped operands.
///
/// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);

/// Convert Op, which must be of float type, to the
/// float type VT, by either extending or rounding (by truncation).
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT);

/// Convert Op, which must be a STRICT operation of float type, to the
/// float type VT, by either extending or rounding (by truncation).
std::pair<SDValue, SDValue>
getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT);

/// Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode.
static unsigned getOpcode_EXTEND(unsigned Opcode) {
  switch (Opcode) {
  case ISD::ANY_EXTEND:
  case ISD::ANY_EXTEND_VECTOR_INREG:
    return ISD::ANY_EXTEND;
  case ISD::ZERO_EXTEND:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
    return ISD::ZERO_EXTEND;
  case ISD::SIGN_EXTEND:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
    return ISD::SIGN_EXTEND;
  }
  llvm_unreachable("Unknown opcode")::llvm::llvm_unreachable_internal("Unknown opcode", "llvm/include/llvm/CodeGen/SelectionDAG.h"
, 920);
}

/// Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode.
static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) {
  switch (Opcode) {
  case ISD::ANY_EXTEND:
  case ISD::ANY_EXTEND_VECTOR_INREG:
    return ISD::ANY_EXTEND_VECTOR_INREG;
  case ISD::ZERO_EXTEND:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
    return ISD::ZERO_EXTEND_VECTOR_INREG;
  case ISD::SIGN_EXTEND:
  case ISD::SIGN_EXTEND_VECTOR_INREG:
    return ISD::SIGN_EXTEND_VECTOR_INREG;
  }
  llvm_unreachable("Unknown opcode")::llvm::llvm_unreachable_internal("Unknown opcode", "llvm/include/llvm/CodeGen/SelectionDAG.h"
, 936);
}

/// Convert Op, which must be of integer type, to the
/// integer type VT, by either any-extending or truncating it.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);

/// Convert Op, which must be of integer type, to the
/// integer type VT, by either sign-extending or truncating it.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);

/// Convert Op, which must be of integer type, to the
/// integer type VT, by either zero-extending or truncating it.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);

/// Return the expression required to zero extend the Op
/// value assuming it was the smaller SrcTy value.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);

/// Convert Op, which must be of integer type, to the integer type VT, by
/// either truncating it or performing either zero or sign extension as
/// appropriate extension for the pointer's semantics.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);

/// Return the expression required to extend the Op as a pointer value
/// assuming it was the smaller SrcTy value. This may be either a zero extend
/// or a sign extend.
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);

/// Convert Op, which must be of integer type, to the integer type VT,
/// by using an extension appropriate for the target's
/// BooleanContent for type OpVT or truncating it.
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);

/// Create negative operation as (SUB 0, Val).
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT);

/// Create a bitwise NOT operation as (XOR Val, -1).
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);

/// Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);

/// Create a vector-predicated logical NOT operation as (VP_XOR Val,
/// BooleanOne, Mask, EVL).
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask,
                        SDValue EVL, EVT VT);

/// Convert a vector-predicated Op, which must be an integer vector, to the
/// vector-type VT, by performing either vector-predicated zext or truncating
/// it. The Op will be returned as-is if Op and VT are vectors containing
/// integer with same width.
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask,
                         SDValue EVL);

/// Convert a vector-predicated Op, which must be of integer type, to the
/// vector-type integer type VT, by either truncating it or performing either
/// vector-predicated zero or sign extension as appropriate extension for the
/// pointer's semantics. This function just redirects to getVPZExtOrTrunc
/// right now.
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask,
                           SDValue EVL);

/// Returns sum of the base pointer and offset.
/// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL,
                             const SDNodeFlags Flags = SDNodeFlags());
SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL,
                             const SDNodeFlags Flags = SDNodeFlags());

/// Create an add instruction with appropriate flags when used for
/// addressing some offset of an object. i.e. if a load is split into multiple
/// components, create an add nuw from the base pointer to the offset.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset) {
  SDNodeFlags Flags;
  Flags.setNoUnsignedWrap(true);
  return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
}

SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) {
  // The object itself can't wrap around the address space, so it shouldn't be
  // possible for the adds of the offsets to the split parts to overflow.
  SDNodeFlags Flags;
  Flags.setNoUnsignedWrap(true);
  return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
}

/// Return a new CALLSEQ_START node, that starts new call frame, in which
/// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and
/// OutSize specifies part of the frame set up prior to the sequence.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize,
                         const SDLoc &DL) {
  SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
  SDValue Ops[] = { Chain,
                    getIntPtrConstant(InSize, DL, true),
                    getIntPtrConstant(OutSize, DL, true) };
  return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
}

/// Return a new CALLSEQ_END node, which always must have a
/// glue result (to ensure it's not CSE'd).
/// CALLSEQ_END does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
                       SDValue InGlue, const SDLoc &DL) {
  SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue);
  SmallVector<SDValue, 4> Ops;
  Ops.push_back(Chain);
  Ops.push_back(Op1);
  Ops.push_back(Op2);
  if (InGlue.getNode())
    Ops.push_back(InGlue);
  return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
}

SDValue getCALLSEQ_END(SDValue Chain, uint64_t Size1, uint64_t Size2,
                       SDValue Glue, const SDLoc &DL) {
  return getCALLSEQ_END(
      Chain, getIntPtrConstant(Size1, DL, /*isTarget=*/true),
      getIntPtrConstant(Size2, DL, /*isTarget=*/true), Glue, DL);
}

/// Return true if the result of this operation is always undefined.
bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops);

/// Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getUNDEF(EVT VT) {
  return getNode(ISD::UNDEF, SDLoc(), VT);
}

/// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
                  bool ConstantFold = true);

/// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
  return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
}

/// Gets or creates the specified node.
///
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                ArrayRef<SDUse> Ops);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
                ArrayRef<SDValue> Ops);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                ArrayRef<SDValue> Ops, const SDNodeFlags Flags);

// Use flags from current flag inserter.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                ArrayRef<SDValue> Ops);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                ArrayRef<SDValue> Ops);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                SDValue N2);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                SDValue N2, SDValue N3);

// Specialize based on number of operands.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
                const SDNodeFlags Flags);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                SDValue N2, const SDNodeFlags Flags);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                SDValue N2, SDValue N3, const SDNodeFlags Flags);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                SDValue N2, SDValue N3, SDValue N4);
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                SDValue N2, SDValue N3, SDValue N4, SDValue N5);

// Specialize again based on number of operands for nodes with a VTList
// rather than a single VT.
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
                SDValue N2);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
                SDValue N2, SDValue N3);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
                SDValue N2, SDValue N3, SDValue N4);
SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
                SDValue N2, SDValue N3, SDValue N4, SDValue N5);

/// Compute a TokenFactor to force all the incoming stack arguments to be
/// loaded from the stack. This is used in tail call lowering to protect
/// stack arguments from being clobbered.
SDValue getStackArgumentTokenFactor(SDValue Chain);

SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
                  SDValue Size, Align Alignment, bool isVol,
                  bool AlwaysInline, bool isTailCall,
                  MachinePointerInfo DstPtrInfo,
                  MachinePointerInfo SrcPtrInfo,
                  const AAMDNodes &AAInfo = AAMDNodes(),
                  AAResults *AA = nullptr);

SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
                   SDValue Size, Align Alignment, bool isVol, bool isTailCall,
                   MachinePointerInfo DstPtrInfo,
                   MachinePointerInfo SrcPtrInfo,
                   const AAMDNodes &AAInfo = AAMDNodes(),
                   AAResults *AA = nullptr);

SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
                  SDValue Size, Align Alignment, bool isVol,
                  bool AlwaysInline, bool isTailCall,
                  MachinePointerInfo DstPtrInfo,
                  const AAMDNodes &AAInfo = AAMDNodes());

SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
                        SDValue Src, SDValue Size, Type *SizeTy,
                        unsigned ElemSz, bool isTailCall,
                        MachinePointerInfo DstPtrInfo,
                        MachinePointerInfo SrcPtrInfo);

SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
                         SDValue Src, SDValue Size, Type *SizeTy,
                         unsigned ElemSz, bool isTailCall,
                         MachinePointerInfo DstPtrInfo,
                         MachinePointerInfo SrcPtrInfo);

SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
                        SDValue Value, SDValue Size, Type *SizeTy,
                        unsigned ElemSz, bool isTailCall,
                        MachinePointerInfo DstPtrInfo);

/// Helper function to make it easier to build SetCC's if you just have an
/// ISD::CondCode instead of an SDValue.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
                 ISD::CondCode Cond, SDValue Chain = SDValue(),
                 bool IsSignaling = false) {
  assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&(static_cast <bool> (LHS.getValueType().isVector() == RHS
.getValueType().isVector() && "Vector/scalar operand type mismatch for setcc"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Vector/scalar operand type mismatch for setcc\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1171, __extension__
 __PRETTY_FUNCTION__))
         "Vector/scalar operand type mismatch for setcc")(static_cast <bool> (LHS.getValueType().isVector() == RHS
.getValueType().isVector() && "Vector/scalar operand type mismatch for setcc"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Vector/scalar operand type mismatch for setcc\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1171, __extension__
 __PRETTY_FUNCTION__));
  assert(LHS.getValueType().isVector() == VT.isVector() &&(static_cast <bool> (LHS.getValueType().isVector() == VT
.isVector() && "Vector/scalar result type mismatch for setcc"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Vector/scalar result type mismatch for setcc\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1173, __extension__
 __PRETTY_FUNCTION__))
         "Vector/scalar result type mismatch for setcc")(static_cast <bool> (LHS.getValueType().isVector() == VT
.isVector() && "Vector/scalar result type mismatch for setcc"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Vector/scalar result type mismatch for setcc\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1173, __extension__
 __PRETTY_FUNCTION__));
  assert(Cond != ISD::SETCC_INVALID &&(static_cast <bool> (Cond != ISD::SETCC_INVALID &&
 "Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail
 ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1175, __extension__
 __PRETTY_FUNCTION__))
         "Cannot create a setCC of an invalid node.")(static_cast <bool> (Cond != ISD::SETCC_INVALID &&
 "Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail
 ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1175, __extension__
 __PRETTY_FUNCTION__));
  if (Chain)
    return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL,
                   {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)});
  return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
}

/// Helper function to make it easier to build VP_SETCCs if you just have an
/// ISD::CondCode instead of an SDValue.
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
                   ISD::CondCode Cond, SDValue Mask, SDValue EVL) {
  assert(LHS.getValueType().isVector() && RHS.getValueType().isVector() &&(static_cast <bool> (LHS.getValueType().isVector() &&
 RHS.getValueType().isVector() && "Cannot compare scalars"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() && RHS.getValueType().isVector() && \"Cannot compare scalars\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1187, __extension__
 __PRETTY_FUNCTION__))
         "Cannot compare scalars")(static_cast <bool> (LHS.getValueType().isVector() &&
 RHS.getValueType().isVector() && "Cannot compare scalars"
) ? void (0) : __assert_fail ("LHS.getValueType().isVector() && RHS.getValueType().isVector() && \"Cannot compare scalars\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1187, __extension__
 __PRETTY_FUNCTION__));
  assert(Cond != ISD::SETCC_INVALID &&(static_cast <bool> (Cond != ISD::SETCC_INVALID &&
 "Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail
 ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1189, __extension__
 __PRETTY_FUNCTION__))
         "Cannot create a setCC of an invalid node.")(static_cast <bool> (Cond != ISD::SETCC_INVALID &&
 "Cannot create a setCC of an invalid node.") ? void (0) : __assert_fail
 ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1189, __extension__
 __PRETTY_FUNCTION__));
  return getNode(ISD::VP_SETCC, DL, VT, LHS, RHS, getCondCode(Cond), Mask,
                 EVL);
}

/// Helper function to make it easier to build Select's if you just have
/// operands and don't want to check for vector.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
                  SDValue RHS) {
  assert(LHS.getValueType() == VT && RHS.getValueType() == VT &&(static_cast <bool> (LHS.getValueType() == VT &&
 RHS.getValueType() == VT && "Cannot use select on differing types"
) ? void (0) : __assert_fail ("LHS.getValueType() == VT && RHS.getValueType() == VT && \"Cannot use select on differing types\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1199, __extension__
 __PRETTY_FUNCTION__))
         "Cannot use select on differing types")(static_cast <bool> (LHS.getValueType() == VT &&
 RHS.getValueType() == VT && "Cannot use select on differing types"
) ? void (0) : __assert_fail ("LHS.getValueType() == VT && RHS.getValueType() == VT && \"Cannot use select on differing types\""
, "llvm/include/llvm/CodeGen/SelectionDAG.h", 1199, __extension__
 __PRETTY_FUNCTION__));
  auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
  return getNode(Opcode, DL, VT, Cond, LHS, RHS);
}

/// Helper function to make it easier to build SelectCC's if you just have an
/// ISD::CondCode instead of an SDValue.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
                    SDValue False, ISD::CondCode Cond) {
  return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True,
                 False, getCondCode(Cond));
}

/// Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);

/// Try to simplify a shift into 1 of its operands or a constant.
SDValue simplifyShift(SDValue X, SDValue Y);

/// Try to simplify a floating-point binary operation into 1 of its operands
/// or a constant.
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
                        SDNodeFlags Flags);

/// VAArg produces a result and token chain, and takes a pointer
/// and a source value as input.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
                 SDValue SV, unsigned Align);

/// Gets a node for an atomic cmpxchg op. There are two
/// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a
/// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
/// a success flag (initially i1), and a chain.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                         SDVTList VTs, SDValue Chain, SDValue Ptr,
                         SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);

/// Gets a node for an atomic op, produces result (if relevant)
/// and chain and takes 2 operands.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
                  SDValue Ptr, SDValue Val, MachineMemOperand *MMO);

/// Gets a node for an atomic op, produces result and chain and
/// takes 1 operand.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
                  SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);

/// Gets a node for an atomic op, produces result and chain and takes N
/// operands.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                  SDVTList VTList, ArrayRef<SDValue> Ops,
                  MachineMemOperand *MMO);

/// Creates a MemIntrinsicNode that may produce a
/// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
/// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
/// less than FIRST_TARGET_MEMORY_OPCODE.
SDValue getMemIntrinsicNode(
    unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
    EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
    MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
                                     MachineMemOperand::MOStore,
    uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes());

inline SDValue getMemIntrinsicNode(
    unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
    EVT MemVT, MachinePointerInfo PtrInfo,
    MaybeAlign Alignment = std::nullopt,
    MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad |
                                     MachineMemOperand::MOStore,
    uint64_t Size = 0, const AAMDNodes &AAInfo = AAMDNodes()) {
  // Ensure that codegen never sees alignment 0
  return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, PtrInfo,
                             Alignment.value_or(getEVTAlign(MemVT)), Flags,
                             Size, AAInfo);
}

SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
                            ArrayRef<SDValue> Ops, EVT MemVT,
                            MachineMemOperand *MMO);

/// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends
/// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between
/// offsets `Offset` and `Offset + Size`.
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
                        int FrameIndex, int64_t Size, int64_t Offset = -1);

/// Creates a PseudoProbeSDNode with function GUID `Guid` and
/// the index of the block `Index` it is probing, as well as the attributes
/// `attr` of the probe.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid,
                           uint64_t Index, uint32_t Attr);

/// Create a MERGE_VALUES node from the given operands.
SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);

/// Loads are not normal binary operators: their result type is not
/// determined by their operands, and they produce a value AND a token chain.
///
/// This function will set the MOLoad flag on MMOFlags, but you can set it if
/// you want.  The MOStore flag must not be set.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
                MachinePointerInfo PtrInfo,
                MaybeAlign Alignment = MaybeAlign(),
                MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
                const AAMDNodes &AAInfo = AAMDNodes(),
                const MDNode *Ranges = nullptr);
/// FIXME: Remove once transition to Align is over.
LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead",__attribute__((deprecated("Use the getLoad function that takes a MaybeAlign instead"
, "")))
                "")__attribute__((deprecated("Use the getLoad function that takes a MaybeAlign instead"
, "")))
inline SDValue
getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
        MachinePointerInfo PtrInfo, unsigned Alignment,
        MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
        const AAMDNodes &AAInfo = AAMDNodes(),
        const MDNode *Ranges = nullptr) {
  return getLoad(VT, dl, Chain, Ptr, PtrInfo, MaybeAlign(Alignment), MMOFlags,
                 AAInfo, Ranges);
}
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
                MachineMemOperand *MMO);
SDValue
getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
           SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
           MaybeAlign Alignment = MaybeAlign(),
           MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
           const AAMDNodes &AAInfo = AAMDNodes());
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
                   SDValue Chain, SDValue Ptr, EVT MemVT,
                   MachineMemOperand *MMO);
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
                       SDValue Offset, ISD::MemIndexedMode AM);
SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
                const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
                MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
                MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
                const AAMDNodes &AAInfo = AAMDNodes(),
                const MDNode *Ranges = nullptr);
inline SDValue getLoad(
    ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
    SDValue Chain, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo,
    EVT MemVT, MaybeAlign Alignment = MaybeAlign(),
    MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
    const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr) {
  // Ensures that codegen never sees a None Alignment.
  return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
                 Alignment.value_or(getEVTAlign(MemVT)), MMOFlags, AAInfo,
                 Ranges);
}
/// FIXME: Remove once transition to Align is over.
LLVM_DEPRECATED("Use the getLoad function that takes a MaybeAlign instead",__attribute__((deprecated("Use the getLoad function that takes a MaybeAlign instead"
, "")))
                "")__attribute__((deprecated("Use the getLoad function that takes a MaybeAlign instead"
, "")))
inline SDValue
getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
        const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
        MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment,
        MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
        const AAMDNodes &AAInfo = AAMDNodes(),
        const MDNode *Ranges = nullptr) {
  return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, PtrInfo, MemVT,
                 MaybeAlign(Alignment), MMOFlags, AAInfo, Ranges);
}
SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
                const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
                EVT MemVT, MachineMemOperand *MMO);

/// Helper function to build ISD::STORE nodes.
///
/// This function will set the MOStore flag on MMOFlags, but you can set it if
/// you want.  The MOLoad and MOInvariant flags must not be set.

SDValue
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
         MachinePointerInfo PtrInfo, Align Alignment,
         MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
         const AAMDNodes &AAInfo = AAMDNodes());
inline SDValue
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
         MachinePointerInfo PtrInfo, MaybeAlign Alignment = MaybeAlign(),
         MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
         const AAMDNodes &AAInfo = AAMDNodes()) {
  return getStore(Chain, dl, Val, Ptr, PtrInfo,
                  Alignment.value_or(getEVTAlign(Val.getValueType())),
31
←
Calling 'SDValue::getValueType'→
                  MMOFlags, AAInfo);
}
/// FIXME: Remove once transition to Align is over.
LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "")__attribute__((deprecated("Use the version that takes a MaybeAlign instead"
, "")))
inline SDValue
getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
         MachinePointerInfo PtrInfo, unsigned Alignment,
         MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
         const AAMDNodes &AAInfo = AAMDNodes()) {
  return getStore(Chain, dl, Val, Ptr, PtrInfo, MaybeAlign(Alignment),
                  MMOFlags, AAInfo);
}
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
                 MachineMemOperand *MMO);
SDValue
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
              MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
              MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
              const AAMDNodes &AAInfo = AAMDNodes());
inline SDValue
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
              MachinePointerInfo PtrInfo, EVT SVT,
              MaybeAlign Alignment = MaybeAlign(),
              MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
              const AAMDNodes &AAInfo = AAMDNodes()) {
  return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
                       Alignment.value_or(getEVTAlign(SVT)), MMOFlags,
                       AAInfo);
}
/// FIXME: Remove once transition to Align is over.
LLVM_DEPRECATED("Use the version that takes a MaybeAlign instead", "")__attribute__((deprecated("Use the version that takes a MaybeAlign instead"
, "")))
inline SDValue
getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
              MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment,
              MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
              const AAMDNodes &AAInfo = AAMDNodes()) {
  return getTruncStore(Chain, dl, Val, Ptr, PtrInfo, SVT,
                       MaybeAlign(Alignment), MMOFlags, AAInfo);
}
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                      SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
                        SDValue Offset, ISD::MemIndexedMode AM);

SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
                  const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
                  SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo,
                  EVT MemVT, Align Alignment,
                  MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
                  const MDNode *Ranges = nullptr, bool IsExpanding = false);
inline SDValue
getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
          const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
          SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT,
          MaybeAlign Alignment = MaybeAlign(),
          MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
          const AAMDNodes &AAInfo = AAMDNodes(),
          const MDNode *Ranges = nullptr, bool IsExpanding = false) {
  // Ensures that codegen never sees a None Alignment.
  return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL,
                   PtrInfo, MemVT, Alignment.value_or(getEVTAlign(MemVT)),
                   MMOFlags, AAInfo, Ranges, IsExpanding);
}
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
                  const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
                  SDValue Mask, SDValue EVL, EVT MemVT,
                  MachineMemOperand *MMO, bool IsExpanding = false);
SDValue getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
                  SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo,
                  MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags,
                  const AAMDNodes &AAInfo, const MDNode *Ranges = nullptr,
                  bool IsExpanding = false);
SDValue getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
                  SDValue Mask, SDValue EVL, MachineMemOperand *MMO,
                  bool IsExpanding = false);
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
                     SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL,
                     MachinePointerInfo PtrInfo, EVT MemVT,
                     MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags,
                     const AAMDNodes &AAInfo, bool IsExpanding = false);
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
                     SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL,
                     EVT MemVT, MachineMemOperand *MMO,
                     bool IsExpanding = false);
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
                         SDValue Offset, ISD::MemIndexedMode AM);
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
                   SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT,
                   MachineMemOperand *MMO, ISD::MemIndexedMode AM,
                   bool IsTruncating = false, bool IsCompressing = false);
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
                        SDValue Ptr, SDValue Mask, SDValue EVL,
                        MachinePointerInfo PtrInfo, EVT SVT, Align Alignment,
                        MachineMemOperand::Flags MMOFlags,
                        const AAMDNodes &AAInfo, bool IsCompressing = false);
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
                        SDValue Ptr, SDValue Mask, SDValue EVL, EVT SVT,
                        MachineMemOperand *MMO, bool IsCompressing = false);
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base,
                          SDValue Offset, ISD::MemIndexedMode AM);

SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                         EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr,
                         SDValue Offset, SDValue Stride, SDValue Mask,
                         SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT,
                         Align Alignment, MachineMemOperand::Flags MMOFlags,
                         const AAMDNodes &AAInfo,
                         const MDNode *Ranges = nullptr,
                         bool IsExpanding = false);
inline SDValue getStridedLoadVP(
    ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
    SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
    SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT,
    MaybeAlign Alignment = MaybeAlign(),
    MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
    const AAMDNodes &AAInfo = AAMDNodes(), const MDNode *Ranges = nullptr,
    bool IsExpanding = false) {
  // Ensures that codegen never sees a None Alignment.
  return getStridedLoadVP(AM, ExtType, VT, DL, Chain, Ptr, Offset, Stride,
                          Mask, EVL, PtrInfo, MemVT,
                          Alignment.value_or(getEVTAlign(MemVT)), MMOFlags,
                          AAInfo, Ranges, IsExpanding);
}
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                         EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr,
                         SDValue Offset, SDValue Stride, SDValue Mask,
                         SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
                         bool IsExpanding = false);
SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr,
                         SDValue Stride, SDValue Mask, SDValue EVL,
                         MachinePointerInfo PtrInfo, MaybeAlign Alignment,
                         MachineMemOperand::Flags MMOFlags,
                         const AAMDNodes &AAInfo,
                         const MDNode *Ranges = nullptr,
                         bool IsExpanding = false);
SDValue getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr,
                         SDValue Stride, SDValue Mask, SDValue EVL,
                         MachineMemOperand *MMO, bool IsExpanding = false);
SDValue
getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT,
                    SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask,
                    SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT,
                    MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags,
                    const AAMDNodes &AAInfo, bool IsExpanding = false);
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT,
                            SDValue Chain, SDValue Ptr, SDValue Stride,
                            SDValue Mask, SDValue EVL, EVT MemVT,
                            MachineMemOperand *MMO, bool IsExpanding = false);
SDValue getIndexedStridedLoadVP(SDValue OrigLoad, const SDLoc &DL,
                                SDValue Base, SDValue Offset,
                                ISD::MemIndexedMode AM);
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val,
                          SDValue Ptr, SDValue Offset, SDValue Stride,
                          SDValue Mask, SDValue EVL, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexedMode AM,
                          bool IsTruncating = false,
                          bool IsCompressing = false);
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val,
                               SDValue Ptr, SDValue Stride, SDValue Mask,
                               SDValue EVL, MachinePointerInfo PtrInfo,
                               EVT SVT, Align Alignment,
                               MachineMemOperand::Flags MMOFlags,
                               const AAMDNodes &AAInfo,
                               bool IsCompressing = false);
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val,
                               SDValue Ptr, SDValue Stride, SDValue Mask,
                               SDValue EVL, EVT SVT, MachineMemOperand *MMO,
                               bool IsCompressing = false);
SDValue getIndexedStridedStoreVP(SDValue OrigStore, const SDLoc &DL,
                                 SDValue Base, SDValue Offset,
                                 ISD::MemIndexedMode AM);

SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
                    ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType);
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
                     ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                     ISD::MemIndexType IndexType);

SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base,
                      SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT,
                      MachineMemOperand *MMO, ISD::MemIndexedMode AM,
                      ISD::LoadExtType, bool IsExpanding = false);
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
                             SDValue Offset, ISD::MemIndexedMode AM);
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
                       SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT,
                       MachineMemOperand *MMO, ISD::MemIndexedMode AM,
                       bool IsTruncating = false, bool IsCompressing = false);
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
                              SDValue Base, SDValue Offset,
                              ISD::MemIndexedMode AM);
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
                        ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                        ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy);
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
                         ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                         ISD::MemIndexType IndexType,
                         bool IsTruncating = false);

/// Construct a node to track a Value* through the backend.
SDValue getSrcValue(const Value *v);

/// Return an MDNodeSDNode which holds an MDNode.
SDValue getMDNode(const MDNode *MD);

/// Return a bitcast using the SDLoc of the value operand, and casting to the
/// provided type. Use getNode to set a custom SDLoc.
SDValue getBitcast(EVT VT, SDValue V);

/// Return an AddrSpaceCastSDNode.
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
                         unsigned DestAS);

/// Return a freeze using the SDLoc of the value operand.
SDValue getFreeze(SDValue V);

/// Return an AssertAlignSDNode.
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A);

/// Swap N1 and N2 if Opcode is a commutative binary opcode
/// and the canonical form expects the opposite order.
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
                                  SDValue &N2) const;

/// Return the specified value casted to
/// the target's desired shift amount type.
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);

/// Expand the specified \c ISD::VAARG node as the Legalize pass would.
SDValue expandVAArg(SDNode *Node);

/// Expand the specified \c ISD::VACOPY node as the Legalize pass would.
SDValue expandVACopy(SDNode *Node);

/// Returs an GlobalAddress of the function from the current module with
/// name matching the given ExternalSymbol. Additionally can provide the
/// matched function.
/// Panics the function doesn't exists.
SDValue getSymbolFunctionGlobalAddress(SDValue Op,
                                       Function **TargetFunction = nullptr);

/// *Mutate* the specified node in-place to have the
/// specified operands.  If the resultant node already exists in the DAG,
/// this does not modify the specified node, instead it returns the node that
/// already exists.  If the resultant node does not exist in the DAG, the
/// input node is returned.  As a degenerate case, if you specify the same
/// input operands as the node already has, the input node is returned.
SDNode *UpdateNodeOperands(SDNode *N, SDValue Op);
SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2);
SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
                             SDValue Op3);
SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
                             SDValue Op3, SDValue Op4);
SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
                             SDValue Op3, SDValue Op4, SDValue Op5);
SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);

/// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k
/// values or more, move values into new TokenFactors in 64k-1 blocks, until
/// the final TokenFactor has less than 64k operands.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);

/// *Mutate* the specified machine node's memory references to the provided
/// list.
void setNodeMemRefs(MachineSDNode *N,
                    ArrayRef<MachineMemOperand *> NewMemRefs);

// Calculate divergence of node \p N based on its operands.
bool calculateDivergence(SDNode *N);

// Propagates the change in divergence to users
void updateDivergence(SDNode * N);

/// These are used for target selectors to *mutate* the
/// specified node to have the specified return type, Target opcode, and
/// operands.  Note that target opcodes are stored as
/// ~TargetOpcode in the node opcode field.  The resultant node is returned.
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
                     SDValue Op1, SDValue Op2);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
                     SDValue Op1, SDValue Op2, SDValue Op3);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
                     ArrayRef<SDValue> Ops);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
                     EVT VT2, ArrayRef<SDValue> Ops);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
                     EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
                     EVT VT2, SDValue Op1, SDValue Op2);
SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs,
                     ArrayRef<SDValue> Ops);

/// This *mutates* the specified node to have the specified
/// return type, opcode, and operands.
SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
                    ArrayRef<SDValue> Ops);

/// Mutate the specified strict FP node to its non-strict equivalent,
/// unlinking the node from its chain and dropping the metadata arguments.
/// The node must be a strict FP node.
SDNode *mutateStrictFPToFP(SDNode *Node);

/// These are used for target selectors to create a new node
/// with specified return type(s), MachineInstr opcode, and operands.
///
/// Note that getMachineNode returns the resultant node.  If there is already
/// a node of the specified opcode and operands, it returns that node instead
/// of the current one.
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
                              SDValue Op1);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
                              SDValue Op1, SDValue Op2);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
                              SDValue Op1, SDValue Op2, SDValue Op3);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
                              ArrayRef<SDValue> Ops);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
                              EVT VT2, SDValue Op1, SDValue Op2);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
                              EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
                              EVT VT2, ArrayRef<SDValue> Ops);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
                              EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
                              EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
                              SDValue Op3);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
                              EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl,
                              ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops);
MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs,
                              ArrayRef<SDValue> Ops);

/// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
                               SDValue Operand);

/// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
                              SDValue Operand, SDValue Subreg);

/// Get the specified node if it's already available, or else return NULL.
SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList,
                        ArrayRef<SDValue> Ops, const SDNodeFlags Flags);
SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList,
                        ArrayRef<SDValue> Ops);

/// Check if a node exists without modifying its flags.
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops);

/// Creates a SDDbgValue node.
SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N,
                        unsigned R, bool IsIndirect, const DebugLoc &DL,
                        unsigned O);

/// Creates a constant SDDbgValue node.
SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr,
                                const Value *C, const DebugLoc &DL,
                                unsigned O);

/// Creates a FrameIndex SDDbgValue node.
SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr,
                                  unsigned FI, bool IsIndirect,
                                  const DebugLoc &DL, unsigned O);

/// Creates a FrameIndex SDDbgValue node.
SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr,
                                  unsigned FI,
                                  ArrayRef<SDNode *> Dependencies,
                                  bool IsIndirect, const DebugLoc &DL,
                                  unsigned O);

/// Creates a VReg SDDbgValue node.
SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
                            unsigned VReg, bool IsIndirect,
                            const DebugLoc &DL, unsigned O);

/// Creates a SDDbgValue node from a list of locations.
SDDbgValue *getDbgValueList(DIVariable *Var, DIExpression *Expr,
                            ArrayRef<SDDbgOperand> Locs,
                            ArrayRef<SDNode *> Dependencies, bool IsIndirect,
                            const DebugLoc &DL, unsigned O, bool IsVariadic);

/// Creates a SDDbgLabel node.
SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O);

/// Transfer debug values from one node to another, while optionally
/// generating fragment expressions for split-up values. If \p InvalidateDbg
/// is set, debug values are invalidated after they are transferred.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
                       unsigned SizeInBits = 0, bool InvalidateDbg = true);

/// Remove the specified node from the system. If any of its
/// operands then becomes dead, remove them as well. Inform UpdateListener
/// for each node deleted.
void RemoveDeadNode(SDNode *N);

/// This method deletes the unreachable nodes in the
/// given list, and any nodes that become unreachable as a result.
void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes);

/// Modify anything using 'From' to use 'To' instead.
/// This can cause recursive merging of nodes in the DAG.  Use the first
/// version if 'From' is known to have a single result, use the second
/// if you have two nodes with identical results (or if 'To' has a superset
/// of the results of 'From'), use the third otherwise.
///
/// These methods all take an optional UpdateListener, which (if not null) is
/// informed about nodes that are deleted and modified due to recursive
/// changes in the dag.
///
/// These functions only replace all existing uses. It's possible that as
/// these replacements are being performed, CSE may cause the From node
/// to be given new uses. These new uses of From are left in place, and
/// not automatically transferred to To.
///
void ReplaceAllUsesWith(SDValue From, SDValue To);
void ReplaceAllUsesWith(SDNode *From, SDNode *To);
void ReplaceAllUsesWith(SDNode *From, const SDValue *To);

/// Replace any uses of From with To, leaving
/// uses of other values produced by From.getNode() alone.
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);

/// Like ReplaceAllUsesOfValueWith, but for multiple values at once.
/// This correctly handles the case where
/// there is an overlap between the From values and the To values.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
                                unsigned Num);

/// If an existing load has uses of its chain, create a token factor node with
/// that chain and the new memory node's chain and update users of the old
/// chain to the token factor. This ensures that the new memory node will have
/// the same relative memory dependency position as the old load. Returns the
/// new merged load chain.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain);

/// If an existing load has uses of its chain, create a token factor node with
/// that chain and the new memory node's chain and update users of the old
/// chain to the token factor. This ensures that the new memory node will have
/// the same relative memory dependency position as the old load. Returns the
/// new merged load chain.
SDValue makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, SDValue NewMemOp);

/// Topological-sort the AllNodes list and a
/// assign a unique node id for each node in the DAG based on their
/// topological order. Returns the number of nodes.
unsigned AssignTopologicalOrder();

/// Move node N in the AllNodes list to be immediately
/// before the given iterator Position. This may be used to update the
/// topological ordering when the list of nodes is modified.
void RepositionNode(allnodes_iterator Position, SDNode *N) {
  AllNodes.insert(Position, AllNodes.remove(N));
}

/// Returns an APFloat semantics tag appropriate for the given type. If VT is
/// a vector type, the element semantics are returned.
static const fltSemantics &EVTToAPFloatSemantics(EVT VT) {
  switch (VT.getScalarType().getSimpleVT().SimpleTy) {
  default: llvm_unreachable("Unknown FP format")::llvm::llvm_unreachable_internal("Unknown FP format", "llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1848);
  case MVT::f16:     return APFloat::IEEEhalf();
  case MVT::bf16:    return APFloat::BFloat();
  case MVT::f32:     return APFloat::IEEEsingle();
  case MVT::f64:     return APFloat::IEEEdouble();
  case MVT::f80:     return APFloat::x87DoubleExtended();
  case MVT::f128:    return APFloat::IEEEquad();
  case MVT::ppcf128: return APFloat::PPCDoubleDouble();
  }
}

/// Add a dbg_value SDNode. If SD is non-null that means the
/// value is produced by SD.
void AddDbgValue(SDDbgValue *DB, bool isParameter);

/// Add a dbg_label SDNode.
void AddDbgLabel(SDDbgLabel *DB);

/// Get the debug values which reference the given SDNode.
ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const {
  return DbgInfo->getSDDbgValues(SD);
}

1871public:
/// Return true if there are any SDDbgValue nodes associated
/// with this SelectionDAG.
bool hasDebugValues() const { return !DbgInfo->empty(); }

SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); }
SDDbgInfo::DbgIterator DbgEnd() const  { return DbgInfo->DbgEnd(); }

SDDbgInfo::DbgIterator ByvalParmDbgBegin() const {
  return DbgInfo->ByvalParmDbgBegin();
}
SDDbgInfo::DbgIterator ByvalParmDbgEnd() const {
  return DbgInfo->ByvalParmDbgEnd();
}

SDDbgInfo::DbgLabelIterator DbgLabelBegin() const {
  return DbgInfo->DbgLabelBegin();
}
SDDbgInfo::DbgLabelIterator DbgLabelEnd() const {
  return DbgInfo->DbgLabelEnd();
}

/// To be invoked on an SDNode that is slated to be erased. This
/// function mirrors \c llvm::salvageDebugInfo.
void salvageDebugInfo(SDNode &N);

void dump() const;

/// In most cases this function returns the ABI alignment for a given type,
/// except for illegal vector types where the alignment exceeds that of the
/// stack. In such cases we attempt to break the vector down to a legal type
/// and return the ABI alignment for that instead.
Align getReducedAlign(EVT VT, bool UseABI);

/// Create a stack temporary based on the size in bytes and the alignment
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment);

/// Create a stack temporary, suitable for holding the specified value type.
/// If minAlign is specified, the slot size will have at least that alignment.
SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);

/// Create a stack temporary suitable for holding either of the specified
/// value types.
SDValue CreateStackTemporary(EVT VT1, EVT VT2);

SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
                         const GlobalAddressSDNode *GA,
                         const SDNode *N2);

SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
                               ArrayRef<SDValue> Ops);

/// Fold floating-point operations with 2 operands when both operands are
/// constants and/or undefined.
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
                           SDValue N1, SDValue N2);

/// Constant fold a setcc to true or false.
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
                  const SDLoc &dl);

/// Return true if the sign bit of Op is known to be zero.
/// We use this predicate to simplify operations downstream.
bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;

/// Return true if 'Op & Mask' is known to be zero.  We
/// use this predicate to simplify operations downstream.  Op and Mask are
/// known to be the same type.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
                       unsigned Depth = 0) const;

/// Return true if 'Op & Mask' is known to be zero in DemandedElts.  We
/// use this predicate to simplify operations downstream.  Op and Mask are
/// known to be the same type.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
                       const APInt &DemandedElts, unsigned Depth = 0) const;

/// Return true if 'Op' is known to be zero in DemandedElts.  We
/// use this predicate to simplify operations downstream.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts,
                        unsigned Depth = 0) const;

/// Return true if '(Op & Mask) == Mask'.
/// Op and Mask are known to be the same type.
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
                          unsigned Depth = 0) const;

/// For each demanded element of a vector, see if it is known to be zero.
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts,
                                     unsigned Depth = 0) const;

/// Determine which bits of Op are known to be either zero or one and return
/// them in Known. For vectors, the known bits are those that are shared by
/// every vector element.
/// Targets can implement the computeKnownBitsForTargetNode method in the
/// TargetLowering class to allow target nodes to be understood.
KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const;

/// Determine which bits of Op are known to be either zero or one and return
/// them in Known. The DemandedElts argument allows us to only collect the
/// known bits that are shared by the requested vector elements.
/// Targets can implement the computeKnownBitsForTargetNode method in the
/// TargetLowering class to allow target nodes to be understood.
KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
                           unsigned Depth = 0) const;

/// Used to represent the possible overflow behavior of an operation.
/// Never: the operation cannot overflow.
/// Always: the operation will always overflow.
/// Sometime: the operation may or may not overflow.
enum OverflowKind {
  OFK_Never,
  OFK_Sometime,
  OFK_Always,
};

/// Determine if the result of the signed addition of 2 nodes can overflow.
OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const;

/// Determine if the result of the unsigned addition of 2 nodes can overflow.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const;

/// Determine if the result of the addition of 2 nodes can overflow.
OverflowKind computeOverflowForAdd(bool IsSigned, SDValue N0,
                                   SDValue N1) const {
  return IsSigned ? computeOverflowForSignedAdd(N0, N1)
                  : computeOverflowForUnsignedAdd(N0, N1);
}

/// Determine if the result of the signed sub of 2 nodes can overflow.
OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const;

/// Determine if the result of the unsigned sub of 2 nodes can overflow.
OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const;

/// Determine if the result of the sub of 2 nodes can overflow.
OverflowKind computeOverflowForSub(bool IsSigned, SDValue N0,
                                   SDValue N1) const {
  return IsSigned ? computeOverflowForSignedSub(N0, N1)
                  : computeOverflowForUnsignedSub(N0, N1);
}

/// Test if the given value is known to have exactly one bit set. This differs
/// from computeKnownBits in that it doesn't necessarily determine which bit
/// is set.
bool isKnownToBeAPowerOfTwo(SDValue Val) const;

/// Return the number of times the sign bit of the register is replicated into
/// the other bits. We know that at least 1 bit is always equal to the sign
/// bit (itself), but other cases can give us information. For example,
/// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
/// to each other, so we return 3. Targets can implement the
/// ComputeNumSignBitsForTarget method in the TargetLowering class to allow
/// target nodes to be understood.
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;

/// Return the number of times the sign bit of the register is replicated into
/// the other bits. We know that at least 1 bit is always equal to the sign
/// bit (itself), but other cases can give us information. For example,
/// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
/// to each other, so we return 3. The DemandedElts argument allows
/// us to only collect the minimum sign bits of the requested vector elements.
/// Targets can implement the ComputeNumSignBitsForTarget method in the
/// TargetLowering class to allow target nodes to be understood.
unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
                            unsigned Depth = 0) const;

/// Get the upper bound on bit size for this Value \p Op as a signed integer.
/// i.e.  x == sext(trunc(x to MaxSignedBits) to bitwidth(x)).
/// Similar to the APInt::getSignificantBits function.
/// Helper wrapper to ComputeNumSignBits.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth = 0) const;

/// Get the upper bound on bit size for this Value \p Op as a signed integer.
/// i.e.  x == sext(trunc(x to MaxSignedBits) to bitwidth(x)).
/// Similar to the APInt::getSignificantBits function.
/// Helper wrapper to ComputeNumSignBits.
unsigned ComputeMaxSignificantBits(SDValue Op, const APInt &DemandedElts,
                                   unsigned Depth = 0) const;

/// Return true if this function can prove that \p Op is never poison
/// and, if \p PoisonOnly is false, does not have undef bits.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly = false,
                                      unsigned Depth = 0) const;

/// Return true if this function can prove that \p Op is never poison
/// and, if \p PoisonOnly is false, does not have undef bits. The DemandedElts
/// argument limits the check to the requested vector elements.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, const APInt &DemandedElts,
                                      bool PoisonOnly = false,
                                      unsigned Depth = 0) const;

/// Return true if this function can prove that \p Op is never poison.
bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth = 0) const {
  return isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ true, Depth);
}

/// Return true if this function can prove that \p Op is never poison. The
/// DemandedElts argument limits the check to the requested vector elements.
bool isGuaranteedNotToBePoison(SDValue Op, const APInt &DemandedElts,
                               unsigned Depth = 0) const {
  return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts,
                                          /*PoisonOnly*/ true, Depth);
}

/// Return true if Op can create undef or poison from non-undef & non-poison
/// operands. The DemandedElts argument limits the check to the requested
/// vector elements.
///
/// \p ConsiderFlags controls whether poison producing flags on the
/// instruction are considered.  This can be used to see if the instruction
/// could still introduce undef or poison even without poison generating flags
/// which might be on the instruction.  (i.e. could the result of
/// Op->dropPoisonGeneratingFlags() still create poison or undef)
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
                            bool PoisonOnly = false,
                            bool ConsiderFlags = true,
                            unsigned Depth = 0) const;

/// Return true if Op can create undef or poison from non-undef & non-poison
/// operands.
///
/// \p ConsiderFlags controls whether poison producing flags on the
/// instruction are considered.  This can be used to see if the instruction
/// could still introduce undef or poison even without poison generating flags
/// which might be on the instruction.  (i.e. could the result of
/// Op->dropPoisonGeneratingFlags() still create poison or undef)
bool canCreateUndefOrPoison(SDValue Op, bool PoisonOnly = false,
                            bool ConsiderFlags = true,
                            unsigned Depth = 0) const;

/// Return true if the specified operand is an ISD::ADD with a ConstantSDNode
/// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that
/// is guaranteed to have the same semantics as an ADD. This handles the
/// equivalence:
///     X|Cst == X+Cst iff X&Cst = 0.
bool isBaseWithConstantOffset(SDValue Op) const;

/// Test whether the given SDValue (or all elements of it, if it is a
/// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is
/// known to never be a signaling NaN (it may still be a qNaN).
bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;

/// \returns true if \p Op is known to never be a signaling NaN.
bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
  return isKnownNeverNaN(Op, true, Depth);
}

/// Test whether the given floating point SDValue is known to never be
/// positive or negative zero.
bool isKnownNeverZeroFloat(SDValue Op) const;

/// Test whether the given SDValue is known to contain non-zero value(s).
bool isKnownNeverZero(SDValue Op) const;

/// Test whether two SDValues are known to compare equal. This
/// is true if they are the same value, or if one is negative zero and the
/// other positive zero.
bool isEqualTo(SDValue A, SDValue B) const;

/// Return true if A and B have no common bits set. As an example, this can
/// allow an 'add' to be transformed into an 'or'.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const;

/// Test whether \p V has a splatted value for all the demanded elements.
///
/// On success \p UndefElts will indicate the elements that have UNDEF
/// values instead of the splat value, this is only guaranteed to be correct
/// for \p DemandedElts.
///
/// NOTE: The function will return true for a demanded splat of UNDEF values.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts,
                  unsigned Depth = 0) const;

/// Test whether \p V has a splatted value.
bool isSplatValue(SDValue V, bool AllowUndefs = false) const;

/// If V is a splatted value, return the source vector and its splat index.
SDValue getSplatSourceVector(SDValue V, int &SplatIndex);

/// If V is a splat vector, return its scalar source operand by extracting
/// that element from the source vector. If LegalTypes is true, this method
/// may only return a legally-typed splat value. If it cannot legalize the
/// splatted value it will return SDValue().
SDValue getSplatValue(SDValue V, bool LegalTypes = false);

/// If a SHL/SRA/SRL node \p V has a constant or splat constant shift amount
/// that is less than the element bit-width of the shift node, return it.
const APInt *getValidShiftAmountConstant(SDValue V,
                                         const APInt &DemandedElts) const;

/// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less
/// than the element bit-width of the shift node, return the minimum value.
const APInt *
getValidMinimumShiftAmountConstant(SDValue V,
                                   const APInt &DemandedElts) const;

/// If a SHL/SRA/SRL node \p V has constant shift amounts that are all less
/// than the element bit-width of the shift node, return the maximum value.
const APInt *
getValidMaximumShiftAmountConstant(SDValue V,
                                   const APInt &DemandedElts) const;

/// Match a binop + shuffle pyramid that represents a horizontal reduction
/// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p
/// Extract. The reduction must use one of the opcodes listed in /p
/// CandidateBinOps and on success /p BinOp will contain the matching opcode.
/// Returns the vector that is being reduced on, or SDValue() if a reduction
/// was not matched. If \p AllowPartials is set then in the case of a
/// reduction pattern that only matches the first few stages, the extracted
/// subvector of the start of the reduction is returned.
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
                            ArrayRef<ISD::NodeType> CandidateBinOps,
                            bool AllowPartials = false);

/// Utility function used by legalize and lowering to
/// "unroll" a vector operation by splitting out the scalars and operating
/// on each element individually.  If the ResNE is 0, fully unroll the vector
/// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
/// If the  ResNE is greater than the width of the vector op, unroll the
/// vector op and fill the end of the resulting vector with UNDEFS.
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);

/// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
/// This is a separate function because those opcodes have two results.
std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
                                                   unsigned ResNE = 0);

/// Return true if loads are next to each other and can be
/// merged. Check that both are nonvolatile and if LD is loading
/// 'Bytes' bytes from a location that is 'Dist' units away from the
/// location that the 'Base' load is loading from.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
                                    unsigned Bytes, int Dist) const;

/// Infer alignment of a load / store address. Return std::nullopt if it
/// cannot be inferred.
MaybeAlign InferPtrAlign(SDValue Ptr) const;

/// Split the scalar node with EXTRACT_ELEMENT using the provided VTs and
/// return the low/high part.
std::pair<SDValue, SDValue> SplitScalar(const SDValue &N, const SDLoc &DL,
                                        const EVT &LoVT, const EVT &HiVT);

/// Compute the VTs needed for the low/hi parts of a type
/// which is split (or expanded) into two not necessarily identical pieces.
std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const;

/// Compute the VTs needed for the low/hi parts of a type, dependent on an
/// enveloping VT that has been split into two identical pieces. Sets the
/// HisIsEmpty flag when hi type has zero storage size.
std::pair<EVT, EVT> GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
                                             bool *HiIsEmpty) const;

/// Split the vector with EXTRACT_SUBVECTOR using the provides
/// VTs and return the low/high part.
std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
                                        const EVT &LoVT, const EVT &HiVT);

/// Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
  EVT LoVT, HiVT;
  std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
  return SplitVector(N, DL, LoVT, HiVT);
}

/// Split the explicit vector length parameter of a VP operation.
std::pair<SDValue, SDValue> SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL);

/// Split the node's operand with EXTRACT_SUBVECTOR and
/// return the low/high part.
std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
{
  return SplitVector(N->getOperand(OpNo), SDLoc(N));
}

/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
SDValue WidenVector(const SDValue &N, const SDLoc &DL);

/// Append the extracted elements from Start to Count out of the vector Op in
/// Args. If Count is 0, all of the elements will be extracted. The extracted
/// elements will have type EVT if it is provided, and otherwise their type
/// will be Op's element type.
void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
                           unsigned Start = 0, unsigned Count = 0,
                           EVT EltVT = EVT());

/// Compute the default alignment value for the given type.
Align getEVTAlign(EVT MemoryVT) const;

/// Test whether the given value is a constant int or similar node.
SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const;

/// Test whether the given value is a constant FP or similar node.
SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) const ;

/// \returns true if \p N is any kind of constant or build_vector of
/// constants, int or float. If a vector, it may not necessarily be a splat.
inline bool isConstantValueOfAnyType(SDValue N) const {
  return isConstantIntBuildVectorOrConstantInt(N) ||
         isConstantFPBuildVectorOrConstantFP(N);
}

/// Set CallSiteInfo to be associated with Node.
void addCallSiteInfo(const SDNode *Node, CallSiteInfoImpl &&CallInfo) {
  SDEI[Node].CSInfo = std::move(CallInfo);
}
/// Return CallSiteInfo associated with Node, or a default if none exists.
CallSiteInfo getCallSiteInfo(const SDNode *Node) {
  auto I = SDEI.find(Node);
  return I != SDEI.end() ? std::move(I->second).CSInfo : CallSiteInfo();
}
/// Set HeapAllocSite to be associated with Node.
void addHeapAllocSite(const SDNode *Node, MDNode *MD) {
  SDEI[Node].HeapAllocSite = MD;
}
/// Return HeapAllocSite associated with Node, or nullptr if none exists.
MDNode *getHeapAllocSite(const SDNode *Node) const {
  auto I = SDEI.find(Node);
  return I != SDEI.end() ? I->second.HeapAllocSite : nullptr;
}
/// Set PCSections to be associated with Node.
void addPCSections(const SDNode *Node, MDNode *MD) {
  SDEI[Node].PCSections = MD;
}
/// Return PCSections associated with Node, or nullptr if none exists.
MDNode *getPCSections(const SDNode *Node) const {
  auto It = SDEI.find(Node);
  return It != SDEI.end() ? It->second.PCSections : nullptr;
}
/// Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) {
  if (NoMerge)
    SDEI[Node].NoMerge = NoMerge;
}
/// Return NoMerge info associated with Node.
bool getNoMergeSiteInfo(const SDNode *Node) const {
  auto I = SDEI.find(Node);
  return I != SDEI.end() ? I->second.NoMerge : false;
}

/// Copy extra info associated with one node to another.
void copyExtraInfo(SDNode *From, SDNode *To);

/// Return the current function's default denormal handling kind for the given
/// floating point type.
DenormalMode getDenormalMode(EVT VT) const {
  return MF->getDenormalMode(EVTToAPFloatSemantics(VT));
}

bool shouldOptForSize() const;

/// Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT,
                          SDNodeFlags Flags);

/// Some opcodes may create immediate undefined behavior when used with some
/// values (integer division-by-zero for example). Therefore, these operations
/// are not generally safe to move around or change.
bool isSafeToSpeculativelyExecute(unsigned Opcode) const {
  switch (Opcode) {
  case ISD::SDIV:
  case ISD::SREM:
  case ISD::SDIVREM:
  case ISD::UDIV:
  case ISD::UREM:
  case ISD::UDIVREM:
    return false;
  default:
    return true;
  }
}

2344private:
void InsertNode(SDNode *N);
bool RemoveNodeFromCSEMaps(SDNode *N);
void AddModifiedNodeToCSEMaps(SDNode *N);
SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
                             void *&InsertPos);
SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
                             void *&InsertPos);
SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc);

void DeleteNodeNotInCSEMaps(SDNode *N);
void DeallocateNode(SDNode *N);

void allnodes_clear();

/// Look up the node specified by ID in CSEMap.  If it exists, return it.  If
/// not, return the insertion token that will make insertion faster.  This
/// overload is for nodes other than Constant or ConstantFP, use the other one
/// for those.
SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);

/// Look up the node specified by ID in CSEMap.  If it exists, return it.  If
/// not, return the insertion token that will make insertion faster.  Performs
/// additional processing for constant nodes.
SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL,
                            void *&InsertPos);

/// Maps to auto-CSE operations.
std::vector<CondCodeSDNode*> CondCodeNodes;

std::vector<SDNode*> ValueTypeNodes;
std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes;
StringMap<SDNode*> ExternalSymbols;

std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols;
DenseMap<MCSymbol *, SDNode *> MCSymbols;

FlagInserter *Inserter = nullptr;
2383};

2385template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> {
using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>;

static nodes_iterator nodes_begin(SelectionDAG *G) {
  return nodes_iterator(G->allnodes_begin());
}

static nodes_iterator nodes_end(SelectionDAG *G) {
  return nodes_iterator(G->allnodes_end());
}
2395};

2397} // end namespace llvm

2399#endif // LLVM_CODEGEN_SELECTIONDAG_H