/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

Bug Summary

File:	llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Warning:	line 698, column 32 The result of the '%' expression is undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG -I include -I /build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/build-llvm -ferror-limit 19 -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-11-10-160236-22541-1 -x c++ /build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

→

1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//

13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/DivisionByConstantInfo.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/KnownBits.h"
32#include "llvm/Support/MathExtras.h"
33#include "llvm/Target/TargetLoweringObjectFile.h"
34#include "llvm/Target/TargetMachine.h"
35#include <cctype>
36using namespace llvm;

38/// NOTE: The TargetMachine owns TLOF.
39TargetLowering::TargetLowering(const TargetMachine &tm)
  : TargetLoweringBase(tm) {}

42const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
return nullptr;
44}

46bool TargetLowering::isPositionIndependent() const {
return getTargetMachine().isPositionIndependent();
48}

50/// Check whether a given call node is in tail position within its function. If
51/// so, it sets Chain to the input chain of the tail call.
52bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
                                        SDValue &Chain) const {
const Function &F = DAG.getMachineFunction().getFunction();

// First, check if tail calls have been disabled in this function.
if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
  return false;

// Conservatively require the attributes of the call to match those of
// the return. Ignore following attributes because they don't affect the
// call sequence.
AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
                         Attribute::DereferenceableOrNull, Attribute::NoAlias,
                         Attribute::NonNull})
  CallerAttrs.removeAttribute(Attr);

if (CallerAttrs.hasAttributes())
  return false;

// It's not safe to eliminate the sign / zero extension of the return value.
if (CallerAttrs.contains(Attribute::ZExt) ||
    CallerAttrs.contains(Attribute::SExt))
  return false;

// Check if the only use is a function return node.
return isUsedByReturnOnly(Node, Chain);
79}

81bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
  const uint32_t *CallerPreservedMask,
  const SmallVectorImpl<CCValAssign> &ArgLocs,
  const SmallVectorImpl<SDValue> &OutVals) const {
for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
  const CCValAssign &ArgLoc = ArgLocs[I];
  if (!ArgLoc.isRegLoc())
    continue;
  MCRegister Reg = ArgLoc.getLocReg();
  // Only look at callee saved registers.
  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
    continue;
  // Check that we pass the value used for the caller.
  // (We look for a CopyFromReg reading a virtual register that is used
  //  for the function live-in value of register Reg)
  SDValue Value = OutVals[I];
  if (Value->getOpcode() != ISD::CopyFromReg)
    return false;
  Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
    return false;
}
return true;
104}

106/// Set CallLoweringInfo attribute flags based on a call instruction
107/// and called function attributes.
108void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
                                                   unsigned ArgIdx) {
IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
Alignment = Call->getParamStackAlign(ArgIdx);
IndirectType = nullptr;
assert(IsByVal + IsPreallocated + IsInAlloca <= 1 &&(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
 <= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
 ("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 125, __extension__ __PRETTY_FUNCTION__))
       "multiple ABI attributes?")(static_cast <bool> (IsByVal + IsPreallocated + IsInAlloca
 <= 1 && "multiple ABI attributes?") ? void (0) : __assert_fail
 ("IsByVal + IsPreallocated + IsInAlloca <= 1 && \"multiple ABI attributes?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 125, __extension__ __PRETTY_FUNCTION__));
if (IsByVal) {
  IndirectType = Call->getParamByValType(ArgIdx);
  if (!Alignment)
    Alignment = Call->getParamAlign(ArgIdx);
}
if (IsPreallocated)
  IndirectType = Call->getParamPreallocatedType(ArgIdx);
if (IsInAlloca)
  IndirectType = Call->getParamInAllocaType(ArgIdx);
135}

137/// Generate a libcall taking the given operands as arguments and returning a
138/// result of type RetVT.
139std::pair<SDValue, SDValue>
140TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
                          ArrayRef<SDValue> Ops,
                          MakeLibCallOptions CallOptions,
                          const SDLoc &dl,
                          SDValue InChain) const {
if (!InChain)
  InChain = DAG.getEntryNode();

TargetLowering::ArgListTy Args;
Args.reserve(Ops.size());

TargetLowering::ArgListEntry Entry;
for (unsigned i = 0; i < Ops.size(); ++i) {
  SDValue NewOp = Ops[i];
  Entry.Node = NewOp;
  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
  Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
                                               CallOptions.IsSExt);
  Entry.IsZExt = !Entry.IsSExt;

  if (CallOptions.IsSoften &&
      !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
    Entry.IsSExt = Entry.IsZExt = false;
  }
  Args.push_back(Entry);
}

if (LC == RTLIB::UNKNOWN_LIBCALL)
  report_fatal_error("Unsupported library call operation!");
SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
                                       getPointerTy(DAG.getDataLayout()));

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
TargetLowering::CallLoweringInfo CLI(DAG);
bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
bool zeroExtend = !signExtend;

if (CallOptions.IsSoften &&
    !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
  signExtend = zeroExtend = false;
}

CLI.setDebugLoc(dl)
    .setChain(InChain)
    .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
    .setNoReturn(CallOptions.DoesNotReturn)
    .setDiscardResult(!CallOptions.IsReturnValueUsed)
    .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
    .setSExtResult(signExtend)
    .setZExtResult(zeroExtend);
return LowerCallTo(CLI);
191}

193bool TargetLowering::findOptimalMemOpLowering(
  std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
  unsigned SrcAS, const AttributeList &FuncAttributes) const {
if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
  return false;

EVT VT = getOptimalMemOpType(Op, FuncAttributes);

if (VT == MVT::Other) {
  // Use the largest integer type whose alignment constraints are satisfied.
  // We only need to check DstAlign here as SrcAlign is always greater or
  // equal to DstAlign (or zero).
  VT = MVT::i64;
  if (Op.isFixedDstAlign())
    while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
           !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
      VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
  assert(VT.isInteger())(static_cast <bool> (VT.isInteger()) ? void (0) : __assert_fail
 ("VT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 210, __extension__ __PRETTY_FUNCTION__));

  // Find the largest legal integer type.
  MVT LVT = MVT::i64;
  while (!isTypeLegal(LVT))
    LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
  assert(LVT.isInteger())(static_cast <bool> (LVT.isInteger()) ? void (0) : __assert_fail
 ("LVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 216, __extension__ __PRETTY_FUNCTION__));

  // If the type we've chosen is larger than the largest legal integer type
  // then use that instead.
  if (VT.bitsGT(LVT))
    VT = LVT;
}

unsigned NumMemOps = 0;
uint64_t Size = Op.size();
while (Size) {
  unsigned VTSize = VT.getSizeInBits() / 8;
  while (VTSize > Size) {
    // For now, only use non-vector load / store's for the left-over pieces.
    EVT NewVT = VT;
    unsigned NewVTSize;

    bool Found = false;
    if (VT.isVector() || VT.isFloatingPoint()) {
      NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
      if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
          isSafeMemOpType(NewVT.getSimpleVT()))
        Found = true;
      else if (NewVT == MVT::i64 &&
               isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
               isSafeMemOpType(MVT::f64)) {
        // i64 is usually not legal on 32-bit targets, but f64 may be.
        NewVT = MVT::f64;
        Found = true;
      }
    }

    if (!Found) {
      do {
        NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
        if (NewVT == MVT::i8)
          break;
      } while (!isSafeMemOpType(NewVT.getSimpleVT()));
    }
    NewVTSize = NewVT.getSizeInBits() / 8;

    // If the new VT cannot cover all of the remaining bits, then consider
    // issuing a (or a pair of) unaligned and overlapping load / store.
    bool Fast;
    if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
        allowsMisalignedMemoryAccesses(
            VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
            MachineMemOperand::MONone, &Fast) &&
        Fast)
      VTSize = Size;
    else {
      VT = NewVT;
      VTSize = NewVTSize;
    }
  }

  if (++NumMemOps > Limit)
    return false;

  MemOps.push_back(VT);
  Size -= VTSize;
}

return true;
280}

282/// Soften the operands of a comparison. This code is shared among BR_CC,
283/// SELECT_CC, and SETCC handlers.
284void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
                                       SDValue &NewLHS, SDValue &NewRHS,
                                       ISD::CondCode &CCCode,
                                       const SDLoc &dl, const SDValue OldLHS,
                                       const SDValue OldRHS) const {
SDValue Chain;
return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
                           OldRHS, Chain);
292}

294void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
                                       SDValue &NewLHS, SDValue &NewRHS,
                                       ISD::CondCode &CCCode,
                                       const SDLoc &dl, const SDValue OldLHS,
                                       const SDValue OldRHS,
                                       SDValue &Chain,
                                       bool IsSignaling) const {
// FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
// not supporting it. We can update this code when libgcc provides such
// functions.

assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
 VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__))
       && "Unsupported setcc type!")(static_cast <bool> ((VT == MVT::f32 || VT == MVT::f64 ||
 VT == MVT::f128 || VT == MVT::ppcf128) && "Unsupported setcc type!"
) ? void (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 306, __extension__ __PRETTY_FUNCTION__));

// Expand into one or more soft-fp libcall(s).
RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
bool ShouldInvertCC = false;
switch (CCCode) {
case ISD::SETEQ:
case ISD::SETOEQ:
  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
        (VT == MVT::f64) ? RTLIB::OEQ_F64 :
        (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
  break;
case ISD::SETNE:
case ISD::SETUNE:
  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
        (VT == MVT::f64) ? RTLIB::UNE_F64 :
        (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
  break;
case ISD::SETGE:
case ISD::SETOGE:
  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
        (VT == MVT::f64) ? RTLIB::OGE_F64 :
        (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
  break;
case ISD::SETLT:
case ISD::SETOLT:
  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
        (VT == MVT::f64) ? RTLIB::OLT_F64 :
        (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
  break;
case ISD::SETLE:
case ISD::SETOLE:
  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
        (VT == MVT::f64) ? RTLIB::OLE_F64 :
        (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
  break;
case ISD::SETGT:
case ISD::SETOGT:
  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
        (VT == MVT::f64) ? RTLIB::OGT_F64 :
        (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
  break;
case ISD::SETO:
  ShouldInvertCC = true;
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case ISD::SETUO:
  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
        (VT == MVT::f64) ? RTLIB::UO_F64 :
        (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
  break;
case ISD::SETONE:
  // SETONE = O && UNE
  ShouldInvertCC = true;
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case ISD::SETUEQ:
  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
        (VT == MVT::f64) ? RTLIB::UO_F64 :
        (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
        (VT == MVT::f64) ? RTLIB::OEQ_F64 :
        (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
  break;
default:
  // Invert CC for unordered comparisons
  ShouldInvertCC = true;
  switch (CCCode) {
  case ISD::SETULT:
    LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
          (VT == MVT::f64) ? RTLIB::OGE_F64 :
          (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
    break;
  case ISD::SETULE:
    LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
          (VT == MVT::f64) ? RTLIB::OGT_F64 :
          (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
    break;
  case ISD::SETUGT:
    LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
          (VT == MVT::f64) ? RTLIB::OLE_F64 :
          (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
    break;
  case ISD::SETUGE:
    LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
          (VT == MVT::f64) ? RTLIB::OLT_F64 :
          (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
    break;
  default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 392);
  }
}

// Use the target specific return value for comparions lib calls.
EVT RetVT = getCmpLibcallReturnType();
SDValue Ops[2] = {NewLHS, NewRHS};
TargetLowering::MakeLibCallOptions CallOptions;
EVT OpsVT[2] = { OldLHS.getValueType(),
                 OldRHS.getValueType() };
CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
NewLHS = Call.first;
NewRHS = DAG.getConstant(0, dl, RetVT);

CCCode = getCmpLibcallCC(LC1);
if (ShouldInvertCC) {
  assert(RetVT.isInteger())(static_cast <bool> (RetVT.isInteger()) ? void (0) : __assert_fail
 ("RetVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 409, __extension__ __PRETTY_FUNCTION__));
  CCCode = getSetCCInverse(CCCode, RetVT);
}

if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
  // Update Chain.
  Chain = Call.second;
} else {
  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
  SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
  auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
  CCCode = getCmpLibcallCC(LC2);
  if (ShouldInvertCC)
    CCCode = getSetCCInverse(CCCode, RetVT);
  NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
  if (Chain)
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
                        Call2.second);
  NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
                       Tmp.getValueType(), Tmp, NewLHS);
  NewRHS = SDValue();
}
432}

434/// Return the entry encoding for a jump table in the current function. The
435/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
436unsigned TargetLowering::getJumpTableEncoding() const {
// In non-pic modes, just use the address of a block.
if (!isPositionIndependent())
  return MachineJumpTableInfo::EK_BlockAddress;

// In PIC mode, if the target supports a GPRel32 directive, use it.
if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
  return MachineJumpTableInfo::EK_GPRel32BlockAddress;

// Otherwise, use a label difference.
return MachineJumpTableInfo::EK_LabelDifference32;
447}

449SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
                                               SelectionDAG &DAG) const {
// If our PIC model is GP relative, use the global offset table as the base.
unsigned JTEncoding = getJumpTableEncoding();

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
    (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
  return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

return Table;
459}

461/// This returns the relocation base for the given PIC jumptable, the same as
462/// getPICJumpTableRelocBase, but as an MCExpr.
463const MCExpr *
464TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
                                           unsigned JTI,MCContext &Ctx) const{
// The normal PIC reloc base is the label at the start of the jump table.
return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
468}

470bool
471TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
const TargetMachine &TM = getTargetMachine();
const GlobalValue *GV = GA->getGlobal();

// If the address is not even local to this DSO we will have to load it from
// a got and then add the offset.
if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
  return false;

// If the code is position independent we will have to add a base register.
if (isPositionIndependent())
  return false;

// Otherwise we can do it.
return true;
486}

488//===----------------------------------------------------------------------===//
489//  Optimization Methods
490//===----------------------------------------------------------------------===//

492/// If the specified instruction has a constant integer operand and there are
493/// bits set in that constant that are not demanded, then clear those bits and
494/// return true.
495bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
                                          const APInt &DemandedBits,
                                          const APInt &DemandedElts,
                                          TargetLoweringOpt &TLO) const {
SDLoc DL(Op);
unsigned Opcode = Op.getOpcode();

// Do target-specific constant optimization.
if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
  return TLO.New.getNode();

// FIXME: ISD::SELECT, ISD::SELECT_CC
switch (Opcode) {
default:
  break;
case ISD::XOR:
case ISD::AND:
case ISD::OR: {
  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  if (!Op1C || Op1C->isOpaque())
    return false;

  // If this is a 'not' op, don't touch it because that's a canonical form.
  const APInt &C = Op1C->getAPIntValue();
  if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
    return false;

  if (!C.isSubsetOf(DemandedBits)) {
    EVT VT = Op.getValueType();
    SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
    SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
    return TLO.CombineTo(Op, NewOp);
  }

  break;
}
}

return false;
534}

536bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
                                          const APInt &DemandedBits,
                                          TargetLoweringOpt &TLO) const {
EVT VT = Op.getValueType();
APInt DemandedElts = VT.isVector()
                         ? APInt::getAllOnes(VT.getVectorNumElements())
                         : APInt(1, 1);
return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
544}

546/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
547/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
548/// generalized for targets with other types of implicit widening casts.
549bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
                                    const APInt &Demanded,
                                    TargetLoweringOpt &TLO) const {
assert(Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getNumOperands() == 2 &&
 "ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 553, __extension__ __PRETTY_FUNCTION__))
       "ShrinkDemandedOp only supports binary operators!")(static_cast <bool> (Op.getNumOperands() == 2 &&
 "ShrinkDemandedOp only supports binary operators!") ? void (
0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 553, __extension__ __PRETTY_FUNCTION__));
assert(Op.getNode()->getNumValues() == 1 &&(static_cast <bool> (Op.getNode()->getNumValues() ==
&& "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 555, __extension__ __PRETTY_FUNCTION__))
       "ShrinkDemandedOp only supports nodes with one result!")(static_cast <bool> (Op.getNode()->getNumValues() ==
&& "ShrinkDemandedOp only supports nodes with one result!"
) ? void (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 555, __extension__ __PRETTY_FUNCTION__));

SelectionDAG &DAG = TLO.DAG;
SDLoc dl(Op);

// Early return, as this function cannot handle vector types.
if (Op.getValueType().isVector())
  return false;

// Don't do this if the node has another user, which may require the
// full value.
if (!Op.getNode()->hasOneUse())
  return false;

// Search for the smallest integer type with free casts to and from
// Op's type. For expedience, just check power-of-2 integer types.
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
unsigned DemandedSize = Demanded.getActiveBits();
unsigned SmallVTBits = DemandedSize;
if (!isPowerOf2_32(SmallVTBits))
  SmallVTBits = NextPowerOf2(SmallVTBits);
for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
      TLI.isZExtFree(SmallVT, Op.getValueType())) {
    // We found a type with free casts.
    SDValue X = DAG.getNode(
        Op.getOpcode(), dl, SmallVT,
        DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
        DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
    assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")(static_cast <bool> (DemandedSize <= SmallVTBits &&
 "Narrowed below demanded bits?") ? void (0) : __assert_fail (
"DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 585, __extension__ __PRETTY_FUNCTION__));
    SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
    return TLO.CombineTo(Op, Z);
  }
}
return false;
591}

593bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                        DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                      !DCI.isBeforeLegalizeOps());
KnownBits Known;

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
if (Simplified) {
  DCI.AddToWorklist(Op.getNode());
  DCI.CommitTargetLoweringOpt(TLO);
}
return Simplified;
606}

608bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                        KnownBits &Known,
                                        TargetLoweringOpt &TLO,
                                        unsigned Depth,
                                        bool AssumeSingleUse) const {
EVT VT = Op.getValueType();

// TODO: We can probably do more work on calculating the known bits and
// simplifying the operations for scalable vectors, but for now we just
// bail out.
if (VT.isScalableVector()) {
  // Pretend we don't know anything for now.
  Known = KnownBits(DemandedBits.getBitWidth());
  return false;
}

APInt DemandedElts = VT.isVector()
                         ? APInt::getAllOnes(VT.getVectorNumElements())
                         : APInt(1, 1);
return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
                            AssumeSingleUse);
629}

631// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
632// TODO: Under what circumstances can we create nodes? Constant folding?
633SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  SelectionDAG &DAG, unsigned Depth) const {
// Limit search depth.
if (Depth >= SelectionDAG::MaxRecursionDepth)
1
Assuming 'Depth' is < 'MaxRecursionDepth'→
2
←
Taking false branch→
  return SDValue();

// Ignore UNDEFs.
if (Op.isUndef())
3
←
Calling 'SDValue::isUndef'→
9
←
Returning from 'SDValue::isUndef'→
  return SDValue();

// Not demanding any bits/elts from Op.
if (DemandedBits == 0 || DemandedElts == 0)
10
←
Calling 'APInt::operator=='→
13
←
Returning from 'APInt::operator=='→
14
←
Calling 'APInt::operator=='→
17
←
Returning from 'APInt::operator=='→
18
←
Taking false branch→
  return DAG.getUNDEF(Op.getValueType());

unsigned NumElts = DemandedElts.getBitWidth();
unsigned BitWidth = DemandedBits.getBitWidth();
KnownBits LHSKnown, RHSKnown;
switch (Op.getOpcode()) {
19
←
Control jumps to 'case BITCAST:'  at line 652→
case ISD::BITCAST: {
  SDValue Src = peekThroughBitcasts(Op.getOperand(0));
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Op.getValueType();
  if (SrcVT == DstVT)
20
←
Calling 'EVT::operator=='→
30
←
Returning from 'EVT::operator=='→
31
←
Taking false branch→
    return Src;

  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
  unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
  if (NumSrcEltBits == NumDstEltBits)
32
←
Assuming 'NumSrcEltBits' is not equal to 'NumDstEltBits'→
    if (SDValue V = SimplifyMultipleUseDemandedBits(
            Src, DemandedBits, DemandedElts, DAG, Depth + 1))
      return DAG.getBitcast(DstVT, V);

  // TODO - bigendian once we have test coverage.
  if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
33
←
Calling 'EVT::isVector'→
36
←
Returning from 'EVT::isVector'→
37
←
Assuming the condition is false→
      DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = NumDstEltBits / NumSrcEltBits;
    unsigned NumSrcElts = SrcVT.getVectorNumElements();
    APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
    for (unsigned i = 0; i != Scale; ++i) {
      unsigned Offset = i * NumSrcEltBits;
      APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
      if (!Sub.isZero()) {
        DemandedSrcBits |= Sub;
        for (unsigned j = 0; j != NumElts; ++j)
          if (DemandedElts[j])
            DemandedSrcElts.setBit((j * Scale) + i);
      }
    }

    if (SDValue V = SimplifyMultipleUseDemandedBits(
            Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
      return DAG.getBitcast(DstVT, V);
  }

  // TODO - bigendian once we have test coverage.
  if ((NumSrcEltBits % NumDstEltBits) == 0 &&
38
←
Assuming the condition is true→
43
←
Taking true branch→
      DAG.getDataLayout().isLittleEndian()) {
39
←
Calling 'DataLayout::isLittleEndian'→
42
←
Returning from 'DataLayout::isLittleEndian'→
    unsigned Scale = NumSrcEltBits / NumDstEltBits;
    unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
44
←
Assuming the condition is false→
45
←
'?' condition is false→
    APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
    for (unsigned i = 0; i45.1
'i' is not equal to 'NumElts'
1
'i' is not equal to 'NumElts'
1
'i' is not equal to 'NumElts'
1
'i' is not equal to 'NumElts'
1
'i' is not equal to 'NumElts'
 != NumElts; ++i)
46
←
Loop condition is true.  Entering loop body→
      if (DemandedElts[i]) {
47
←
Calling 'APInt::operator[]'→
51
←
Returning from 'APInt::operator[]'→
52
←
Taking true branch→
        unsigned Offset = (i % Scale) * NumDstEltBits;
53
←
The result of the '%' expression is undefined
        DemandedSrcBits.insertBits(DemandedBits, Offset);
        DemandedSrcElts.setBit(i / Scale);
      }

    if (SDValue V = SimplifyMultipleUseDemandedBits(
            Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
      return DAG.getBitcast(DstVT, V);
  }

  break;
}
case ISD::AND: {
  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

  // If all of the demanded bits are known 1 on one side, return the other.
  // These bits cannot contribute to the result of the 'and' in this
  // context.
  if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
    return Op.getOperand(0);
  if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
    return Op.getOperand(1);
  break;
}
case ISD::OR: {
  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

  // If all of the demanded bits are known zero on one side, return the
  // other.  These bits cannot contribute to the result of the 'or' in this
  // context.
  if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
    return Op.getOperand(0);
  if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
    return Op.getOperand(1);
  break;
}
case ISD::XOR: {
  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

  // If all of the demanded bits are known zero on one side, return the
  // other.
  if (DemandedBits.isSubsetOf(RHSKnown.Zero))
    return Op.getOperand(0);
  if (DemandedBits.isSubsetOf(LHSKnown.Zero))
    return Op.getOperand(1);
  break;
}
case ISD::SHL: {
  // If we are only demanding sign bits then we can use the shift source
  // directly.
  if (const APInt *MaxSA =
          DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
    SDValue Op0 = Op.getOperand(0);
    unsigned ShAmt = MaxSA->getZExtValue();
    unsigned NumSignBits =
        DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
    unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
    if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
      return Op0;
  }
  break;
}
case ISD::SETCC: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
  // If (1) we only need the sign-bit, (2) the setcc operands are the same
  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
  // -1, we may be able to bypass the setcc.
  if (DemandedBits.isSignMask() &&
      Op0.getScalarValueSizeInBits() == BitWidth &&
      getBooleanContents(Op0.getValueType()) ==
          BooleanContent::ZeroOrNegativeOneBooleanContent) {
    // If we're testing X < 0, then this compare isn't needed - just use X!
    // FIXME: We're limiting to integer types here, but this should also work
    // if we don't care about FP signed-zero. The use of SETLT with FP means
    // that we don't care about NaNs.
    if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
        (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
      return Op0;
  }
  break;
}
case ISD::SIGN_EXTEND_INREG: {
  // If none of the extended bits are demanded, eliminate the sextinreg.
  SDValue Op0 = Op.getOperand(0);
  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  unsigned ExBits = ExVT.getScalarSizeInBits();
  if (DemandedBits.getActiveBits() <= ExBits)
    return Op0;
  // If the input is already sign extended, just drop the extension.
  unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
  if (NumSignBits >= (BitWidth - ExBits + 1))
    return Op0;
  break;
}
case ISD::ANY_EXTEND_VECTOR_INREG:
case ISD::SIGN_EXTEND_VECTOR_INREG:
case ISD::ZERO_EXTEND_VECTOR_INREG: {
  // If we only want the lowest element and none of extended bits, then we can
  // return the bitcasted source vector.
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Op.getValueType();
  if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
      DAG.getDataLayout().isLittleEndian() &&
      DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
    return DAG.getBitcast(DstVT, Src);
  }
  break;
}
case ISD::INSERT_VECTOR_ELT: {
  // If we don't demand the inserted element, return the base vector.
  SDValue Vec = Op.getOperand(0);
  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  EVT VecVT = Vec.getValueType();
  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
      !DemandedElts[CIdx->getZExtValue()])
    return Vec;
  break;
}
case ISD::INSERT_SUBVECTOR: {
  SDValue Vec = Op.getOperand(0);
  SDValue Sub = Op.getOperand(1);
  uint64_t Idx = Op.getConstantOperandVal(2);
  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
  // If we don't demand the inserted subvector, return the base vector.
  if (DemandedSubElts == 0)
    return Vec;
  // If this simply widens the lowest subvector, see if we can do it earlier.
  if (Idx == 0 && Vec.isUndef()) {
    if (SDValue NewSub = SimplifyMultipleUseDemandedBits(
            Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))
      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
                         Op.getOperand(0), NewSub, Op.getOperand(2));
  }
  break;
}
case ISD::VECTOR_SHUFFLE: {
  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

  // If all the demanded elts are from one operand and are inline,
  // then we can use the operand directly.
  bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = ShuffleMask[i];
    if (M < 0 || !DemandedElts[i])
      continue;
    AllUndef = false;
    IdentityLHS &= (M == (int)i);
    IdentityRHS &= ((M - NumElts) == i);
  }

  if (AllUndef)
    return DAG.getUNDEF(Op.getValueType());
  if (IdentityLHS)
    return Op.getOperand(0);
  if (IdentityRHS)
    return Op.getOperand(1);
  break;
}
default:
  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
    if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
            Op, DemandedBits, DemandedElts, DAG, Depth))
      return V;
  break;
}
return SDValue();
871}

873SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
  SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
  unsigned Depth) const {
EVT VT = Op.getValueType();
APInt DemandedElts = VT.isVector()
                         ? APInt::getAllOnes(VT.getVectorNumElements())
                         : APInt(1, 1);
return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
                                       Depth);
882}

884SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
  SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
  unsigned Depth) const {
APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
                                       Depth);
890}

892/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
893/// result of Op are ever used downstream. If we can use this information to
894/// simplify Op, create a new simplified DAG node and return true, returning the
895/// original and new nodes in Old and New. Otherwise, analyze the expression and
896/// return a mask of Known bits for the expression (used to simplify the
897/// caller).  The Known bits may only be accurate for those bits in the
898/// OriginalDemandedBits and OriginalDemandedElts.
899bool TargetLowering::SimplifyDemandedBits(
  SDValue Op, const APInt &OriginalDemandedBits,
  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
  unsigned Depth, bool AssumeSingleUse) const {
unsigned BitWidth = OriginalDemandedBits.getBitWidth();
assert(Op.getScalarValueSizeInBits() == BitWidth &&(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
 && "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 905, __extension__ __PRETTY_FUNCTION__))
       "Mask size mismatches value type size!")(static_cast <bool> (Op.getScalarValueSizeInBits() == BitWidth
 && "Mask size mismatches value type size!") ? void (
0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 905, __extension__ __PRETTY_FUNCTION__));

// Don't know anything.
Known = KnownBits(BitWidth);

// TODO: We can probably do more work on calculating the known bits and
// simplifying the operations for scalable vectors, but for now we just
// bail out.
if (Op.getValueType().isScalableVector())
  return false;

unsigned NumElts = OriginalDemandedElts.getBitWidth();
assert((!Op.getValueType().isVector() ||(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
 == Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))
        NumElts == Op.getValueType().getVectorNumElements()) &&(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
 == Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__))
       "Unexpected vector size")(static_cast <bool> ((!Op.getValueType().isVector() || NumElts
 == Op.getValueType().getVectorNumElements()) && "Unexpected vector size"
) ? void (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 919, __extension__ __PRETTY_FUNCTION__));

APInt DemandedBits = OriginalDemandedBits;
APInt DemandedElts = OriginalDemandedElts;
SDLoc dl(Op);
auto &DL = TLO.DAG.getDataLayout();

// Undef operand.
if (Op.isUndef())
  return false;

if (Op.getOpcode() == ISD::Constant) {
  // We know all of the bits for a constant!
  Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
  return false;
}

if (Op.getOpcode() == ISD::ConstantFP) {
  // We know all of the bits for a floating point constant!
  Known = KnownBits::makeConstant(
      cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
  return false;
}

// Other users may use these bits.
EVT VT = Op.getValueType();
if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
  if (Depth != 0) {
    // If not at the root, Just compute the Known bits to
    // simplify things downstream.
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false;
  }
  // If this is the root being simplified, allow it to have multiple uses,
  // just set the DemandedBits/Elts to all bits.
  DemandedBits = APInt::getAllOnes(BitWidth);
  DemandedElts = APInt::getAllOnes(NumElts);
} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
  // Not demanding any bits/elts from Op.
  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
} else if (Depth >= SelectionDAG::MaxRecursionDepth) {
  // Limit search depth.
  return false;
}

KnownBits Known2;
switch (Op.getOpcode()) {
case ISD::TargetConstant:
  llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
 "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 967);
case ISD::SCALAR_TO_VECTOR: {
  if (!DemandedElts[0])
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

  KnownBits SrcKnown;
  SDValue Src = Op.getOperand(0);
  unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
  APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
    return true;

  // Upper elements are undef, so only get the knownbits if we just demand
  // the bottom element.
  if (DemandedElts == 1)
    Known = SrcKnown.anyextOrTrunc(BitWidth);
  break;
}
case ISD::BUILD_VECTOR:
  // Collect the known bits that are shared by every demanded element.
  // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
  return false; // Don't fall through, will infinitely loop.
case ISD::LOAD: {
  auto *LD = cast<LoadSDNode>(Op);
  if (getTargetConstantFromLoad(LD)) {
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false; // Don't fall through, will infinitely loop.
  }
  if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
    // If this is a ZEXTLoad and we are looking at the loaded value.
    EVT MemVT = LD->getMemoryVT();
    unsigned MemBits = MemVT.getScalarSizeInBits();
    Known.Zero.setBitsFrom(MemBits);
    return false; // Don't fall through, will infinitely loop.
  }
  break;
}
case ISD::INSERT_VECTOR_ELT: {
  SDValue Vec = Op.getOperand(0);
  SDValue Scl = Op.getOperand(1);
  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  EVT VecVT = Vec.getValueType();

  // If index isn't constant, assume we need all vector elements AND the
  // inserted element.
  APInt DemandedVecElts(DemandedElts);
  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
    unsigned Idx = CIdx->getZExtValue();
    DemandedVecElts.clearBit(Idx);

    // Inserted element is not required.
    if (!DemandedElts[Idx])
      return TLO.CombineTo(Op, Vec);
  }

  KnownBits KnownScl;
  unsigned NumSclBits = Scl.getScalarValueSizeInBits();
  APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
  if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
    return true;

  Known = KnownScl.anyextOrTrunc(BitWidth);

  KnownBits KnownVec;
  if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
                           Depth + 1))
    return true;

  if (!!DemandedVecElts)
    Known = KnownBits::commonBits(Known, KnownVec);

  return false;
}
case ISD::INSERT_SUBVECTOR: {
  // Demand any elements from the subvector and the remainder from the src its
  // inserted into.
  SDValue Src = Op.getOperand(0);
  SDValue Sub = Op.getOperand(1);
  uint64_t Idx = Op.getConstantOperandVal(2);
  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
  APInt DemandedSrcElts = DemandedElts;
  DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);

  KnownBits KnownSub, KnownSrc;
  if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
                           Depth + 1))
    return true;
  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
                           Depth + 1))
    return true;

  Known.Zero.setAllBits();
  Known.One.setAllBits();
  if (!!DemandedSubElts)
    Known = KnownBits::commonBits(Known, KnownSub);
  if (!!DemandedSrcElts)
    Known = KnownBits::commonBits(Known, KnownSrc);

  // Attempt to avoid multi-use src if we don't need anything from it.
  if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
      !DemandedSrcElts.isAllOnes()) {
    SDValue NewSub = SimplifyMultipleUseDemandedBits(
        Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
    SDValue NewSrc = SimplifyMultipleUseDemandedBits(
        Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
    if (NewSub || NewSrc) {
      NewSub = NewSub ? NewSub : Sub;
      NewSrc = NewSrc ? NewSrc : Src;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
                                      Op.getOperand(2));
      return TLO.CombineTo(Op, NewOp);
    }
  }
  break;
}
case ISD::EXTRACT_SUBVECTOR: {
  // Offset the demanded elts by the subvector index.
  SDValue Src = Op.getOperand(0);
  if (Src.getValueType().isScalableVector())
    break;
  uint64_t Idx = Op.getConstantOperandVal(1);
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
                           Depth + 1))
    return true;

  // Attempt to avoid multi-use src if we don't need anything from it.
  if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
    SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
        Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
    if (DemandedSrc) {
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
                                      Op.getOperand(1));
      return TLO.CombineTo(Op, NewOp);
    }
  }
  break;
}
case ISD::CONCAT_VECTORS: {
  Known.Zero.setAllBits();
  Known.One.setAllBits();
  EVT SubVT = Op.getOperand(0).getValueType();
  unsigned NumSubVecs = Op.getNumOperands();
  unsigned NumSubElts = SubVT.getVectorNumElements();
  for (unsigned i = 0; i != NumSubVecs; ++i) {
    APInt DemandedSubElts =
        DemandedElts.extractBits(NumSubElts, i * NumSubElts);
    if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
                             Known2, TLO, Depth + 1))
      return true;
    // Known bits are shared by every demanded subvector element.
    if (!!DemandedSubElts)
      Known = KnownBits::commonBits(Known, Known2);
  }
  break;
}
case ISD::VECTOR_SHUFFLE: {
  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

  // Collect demanded elements from shuffle operands..
  APInt DemandedLHS(NumElts, 0);
  APInt DemandedRHS(NumElts, 0);
  for (unsigned i = 0; i != NumElts; ++i) {
    if (!DemandedElts[i])
      continue;
    int M = ShuffleMask[i];
    if (M < 0) {
      // For UNDEF elements, we don't know anything about the common state of
      // the shuffle result.
      DemandedLHS.clearAllBits();
      DemandedRHS.clearAllBits();
      break;
    }
    assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
* NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1144, __extension__ __PRETTY_FUNCTION__));
    if (M < (int)NumElts)
      DemandedLHS.setBit(M);
    else
      DemandedRHS.setBit(M - NumElts);
  }

  if (!!DemandedLHS || !!DemandedRHS) {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    Known.Zero.setAllBits();
    Known.One.setAllBits();
    if (!!DemandedLHS) {
      if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
                               Depth + 1))
        return true;
      Known = KnownBits::commonBits(Known, Known2);
    }
    if (!!DemandedRHS) {
      if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
                               Depth + 1))
        return true;
      Known = KnownBits::commonBits(Known, Known2);
    }

    // Attempt to avoid multi-use ops if we don't need anything from them.
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
      return TLO.CombineTo(Op, NewOp);
    }
  }
  break;
}
case ISD::AND: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  // If the RHS is a constant, check to see if the LHS would be zero without
  // using the bits from the RHS.  Below, we use knowledge about the RHS to
  // simplify the LHS, here we're using information from the LHS to simplify
  // the RHS.
  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
    // Do not increment Depth here; that can cause an infinite loop.
    KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
    // If the LHS already has zeros where RHSC does, this 'and' is dead.
    if ((LHSKnown.Zero & DemandedBits) ==
        (~RHSC->getAPIntValue() & DemandedBits))
      return TLO.CombineTo(Op, Op0);

    // If any of the set bits in the RHS are known zero on the LHS, shrink
    // the constant.
    if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
                               DemandedElts, TLO))
      return true;

    // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
    // constant, but if this 'and' is only clearing bits that were just set by
    // the xor, then this 'and' can be eliminated by shrinking the mask of
    // the xor. For example, for a 32-bit X:
    // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
    if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
        LHSKnown.One == ~RHSC->getAPIntValue()) {
      SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
      return TLO.CombineTo(Op, Xor);
    }
  }

  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1221, __extension__ __PRETTY_FUNCTION__));
  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
                           Known2, TLO, Depth + 1))
    return true;
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__));

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If all of the demanded bits are known one on one side, return the other.
  // These bits cannot contribute to the result of the 'and'.
  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
    return TLO.CombineTo(Op, Op0);
  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
    return TLO.CombineTo(Op, Op1);
  // If all of the demanded bits in the inputs are known zeros, return zero.
  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
    return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
  // If the RHS is a constant, see if we can simplify it.
  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
                             TLO))
    return true;
  // If the operation can be done in a smaller type, do so.
  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
    return true;

  Known &= Known2;
  break;
}
case ISD::OR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1268, __extension__ __PRETTY_FUNCTION__));
  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
                           Known2, TLO, Depth + 1))
    return true;
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1272, __extension__ __PRETTY_FUNCTION__));

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If all of the demanded bits are known zero on one side, return the other.
  // These bits cannot contribute to the result of the 'or'.
  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
    return TLO.CombineTo(Op, Op0);
  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
    return TLO.CombineTo(Op, Op1);
  // If the RHS is a constant, see if we can simplify it.
  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
    return true;
  // If the operation can be done in a smaller type, do so.
  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
    return true;

  Known |= Known2;
  break;
}
case ISD::XOR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1311, __extension__ __PRETTY_FUNCTION__));
  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
                           Depth + 1))
    return true;
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1315, __extension__ __PRETTY_FUNCTION__));

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If all of the demanded bits are known zero on one side, return the other.
  // These bits cannot contribute to the result of the 'xor'.
  if (DemandedBits.isSubsetOf(Known.Zero))
    return TLO.CombineTo(Op, Op0);
  if (DemandedBits.isSubsetOf(Known2.Zero))
    return TLO.CombineTo(Op, Op1);
  // If the operation can be done in a smaller type, do so.
  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
    return true;

  // If all of the unknown bits are known to be zero on one side or the other
  // turn this into an *inclusive* or.
  //    e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

  ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
  if (C) {
    // If one side is a constant, and all of the set bits in the constant are
    // also known set on the other side, turn this into an AND, as we know
    // the bits will be cleared.
    //    e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
    // NB: it is okay if more bits are known than are requested
    if (C->getAPIntValue() == Known2.One) {
      SDValue ANDC =
          TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
    }

    // If the RHS is a constant, see if we can change it. Don't alter a -1
    // constant because that's a 'not' op, and that is better for combining
    // and codegen.
    if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
      // We're flipping all demanded bits. Flip the undemanded bits too.
      SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
      return TLO.CombineTo(Op, New);
    }
  }

  // If we can't turn this into a 'not', try to shrink the constant.
  if (!C || !C->isAllOnes())
    if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
      return true;

  Known ^= Known2;
  break;
}
case ISD::SELECT:
  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
                           Depth + 1))
    return true;
  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1385, __extension__ __PRETTY_FUNCTION__));
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1386, __extension__ __PRETTY_FUNCTION__));

  // If the operands are constants, see if we can simplify them.
  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
    return true;

  // Only known if known in both the LHS and RHS.
  Known = KnownBits::commonBits(Known, Known2);
  break;
case ISD::SELECT_CC:
  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
                           Depth + 1))
    return true;
  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1402, __extension__ __PRETTY_FUNCTION__));
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1403, __extension__ __PRETTY_FUNCTION__));

  // If the operands are constants, see if we can simplify them.
  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
    return true;

  // Only known if known in both the LHS and RHS.
  Known = KnownBits::commonBits(Known, Known2);
  break;
case ISD::SETCC: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
  // If (1) we only need the sign-bit, (2) the setcc operands are the same
  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
  // -1, we may be able to bypass the setcc.
  if (DemandedBits.isSignMask() &&
      Op0.getScalarValueSizeInBits() == BitWidth &&
      getBooleanContents(Op0.getValueType()) ==
          BooleanContent::ZeroOrNegativeOneBooleanContent) {
    // If we're testing X < 0, then this compare isn't needed - just use X!
    // FIXME: We're limiting to integer types here, but this should also work
    // if we don't care about FP signed-zero. The use of SETLT with FP means
    // that we don't care about NaNs.
    if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
        (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
      return TLO.CombineTo(Op, Op0);

    // TODO: Should we check for other forms of sign-bit comparisons?
    // Examples: X <= -1, X >= 0
  }
  if (getBooleanContents(Op0.getValueType()) ==
          TargetLowering::ZeroOrOneBooleanContent &&
      BitWidth > 1)
    Known.Zero.setBitsFrom(1);
  break;
}
case ISD::SHL: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  EVT ShiftVT = Op1.getValueType();

  if (const APInt *SA =
          TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
    unsigned ShAmt = SA->getZExtValue();
    if (ShAmt == 0)
      return TLO.CombineTo(Op, Op0);

    // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
    // single shift.  We can do this if the bottom bits (which are shifted
    // out) are never demanded.
    // TODO - support non-uniform vector amounts.
    if (Op0.getOpcode() == ISD::SRL) {
      if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
        if (const APInt *SA2 =
                TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
          unsigned C1 = SA2->getZExtValue();
          unsigned Opc = ISD::SHL;
          int Diff = ShAmt - C1;
          if (Diff < 0) {
            Diff = -Diff;
            Opc = ISD::SRL;
          }
          SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
          return TLO.CombineTo(
              Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
        }
      }
    }

    // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
    // are not demanded. This will likely allow the anyext to be folded away.
    // TODO - support non-uniform vector amounts.
    if (Op0.getOpcode() == ISD::ANY_EXTEND) {
      SDValue InnerOp = Op0.getOperand(0);
      EVT InnerVT = InnerOp.getValueType();
      unsigned InnerBits = InnerVT.getScalarSizeInBits();
      if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
          isTypeDesirableForOp(ISD::SHL, InnerVT)) {
        EVT ShTy = getShiftAmountTy(InnerVT, DL);
        if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
          ShTy = InnerVT;
        SDValue NarrowShl =
            TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
                            TLO.DAG.getConstant(ShAmt, dl, ShTy));
        return TLO.CombineTo(
            Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
      }

      // Repeat the SHL optimization above in cases where an extension
      // intervenes: (shl (anyext (shr x, c1)), c2) to
      // (shl (anyext x), c2-c1).  This requires that the bottom c1 bits
      // aren't demanded (as above) and that the shifted upper c1 bits of
      // x aren't demanded.
      // TODO - support non-uniform vector amounts.
      if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
          InnerOp.hasOneUse()) {
        if (const APInt *SA2 =
                TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
          unsigned InnerShAmt = SA2->getZExtValue();
          if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
              DemandedBits.getActiveBits() <=
                  (InnerBits - InnerShAmt + ShAmt) &&
              DemandedBits.countTrailingZeros() >= ShAmt) {
            SDValue NewSA =
                TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
            SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
                                             InnerOp.getOperand(0));
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
          }
        }
      }
    }

    APInt InDemandedMask = DemandedBits.lshr(ShAmt);
    if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1522, __extension__ __PRETTY_FUNCTION__));
    Known.Zero <<= ShAmt;
    Known.One <<= ShAmt;
    // low bits known zero.
    Known.Zero.setLowBits(ShAmt);

    // Try shrinking the operation as long as the shift amount will still be
    // in range.
    if ((ShAmt < DemandedBits.getActiveBits()) &&
        ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
      return true;
  }

  // If we are only demanding sign bits then we can use the shift source
  // directly.
  if (const APInt *MaxSA =
          TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
    unsigned ShAmt = MaxSA->getZExtValue();
    unsigned NumSignBits =
        TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
    unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
    if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
      return TLO.CombineTo(Op, Op0);
  }
  break;
}
case ISD::SRL: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  EVT ShiftVT = Op1.getValueType();

  if (const APInt *SA =
          TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
    unsigned ShAmt = SA->getZExtValue();
    if (ShAmt == 0)
      return TLO.CombineTo(Op, Op0);

    // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
    // single shift.  We can do this if the top bits (which are shifted out)
    // are never demanded.
    // TODO - support non-uniform vector amounts.
    if (Op0.getOpcode() == ISD::SHL) {
      if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
        if (const APInt *SA2 =
                TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
          unsigned C1 = SA2->getZExtValue();
          unsigned Opc = ISD::SRL;
          int Diff = ShAmt - C1;
          if (Diff < 0) {
            Diff = -Diff;
            Opc = ISD::SHL;
          }
          SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
          return TLO.CombineTo(
              Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
        }
      }
    }

    APInt InDemandedMask = (DemandedBits << ShAmt);

    // If the shift is exact, then it does demand the low bits (and knows that
    // they are zero).
    if (Op->getFlags().hasExact())
      InDemandedMask.setLowBits(ShAmt);

    // Compute the new bits that are at the top now.
    if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1592, __extension__ __PRETTY_FUNCTION__));
    Known.Zero.lshrInPlace(ShAmt);
    Known.One.lshrInPlace(ShAmt);
    // High bits known zero.
    Known.Zero.setHighBits(ShAmt);
  }
  break;
}
case ISD::SRA: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  EVT ShiftVT = Op1.getValueType();

  // If we only want bits that already match the signbit then we don't need
  // to shift.
  unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
  if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
      NumHiDemandedBits)
    return TLO.CombineTo(Op, Op0);

  // If this is an arithmetic shift right and only the low-bit is set, we can
  // always convert this into a logical shr, even if the shift amount is
  // variable.  The low bit of the shift cannot be an input sign bit unless
  // the shift amount is >= the size of the datatype, which is undefined.
  if (DemandedBits.isOne())
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

  if (const APInt *SA =
          TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
    unsigned ShAmt = SA->getZExtValue();
    if (ShAmt == 0)
      return TLO.CombineTo(Op, Op0);

    APInt InDemandedMask = (DemandedBits << ShAmt);

    // If the shift is exact, then it does demand the low bits (and knows that
    // they are zero).
    if (Op->getFlags().hasExact())
      InDemandedMask.setLowBits(ShAmt);

    // If any of the demanded bits are produced by the sign extension, we also
    // demand the input sign bit.
    if (DemandedBits.countLeadingZeros() < ShAmt)
      InDemandedMask.setSignBit();

    if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1640, __extension__ __PRETTY_FUNCTION__));
    Known.Zero.lshrInPlace(ShAmt);
    Known.One.lshrInPlace(ShAmt);

    // If the input sign bit is known to be zero, or if none of the top bits
    // are demanded, turn this into an unsigned shift right.
    if (Known.Zero[BitWidth - ShAmt - 1] ||
        DemandedBits.countLeadingZeros() >= ShAmt) {
      SDNodeFlags Flags;
      Flags.setExact(Op->getFlags().hasExact());
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
    }

    int Log2 = DemandedBits.exactLogBase2();
    if (Log2 >= 0) {
      // The bit must come from the sign.
      SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
    }

    if (Known.One[BitWidth - ShAmt - 1])
      // New bits are known one.
      Known.One.setHighBits(ShAmt);

    // Attempt to avoid multi-use ops if we don't need anything from them.
    if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
      SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
          Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
      if (DemandedOp0) {
        SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
        return TLO.CombineTo(Op, NewOp);
      }
    }
  }
  break;
}
case ISD::FSHL:
case ISD::FSHR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  SDValue Op2 = Op.getOperand(2);
  bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

  if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
    unsigned Amt = SA->getAPIntValue().urem(BitWidth);

    // For fshl, 0-shift returns the 1st arg.
    // For fshr, 0-shift returns the 2nd arg.
    if (Amt == 0) {
      if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
                               Known, TLO, Depth + 1))
        return true;
      break;
    }

    // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
    // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
    APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
    APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
    if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;

    Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
    Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
    Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
    Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
    Known.One |= Known2.One;
    Known.Zero |= Known2.Zero;
  }

  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
  if (isPowerOf2_32(BitWidth)) {
    APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
    if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
                             Known2, TLO, Depth + 1))
      return true;
  }
  break;
}
case ISD::ROTL:
case ISD::ROTR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
  if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
    return TLO.CombineTo(Op, Op0);

  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
  if (isPowerOf2_32(BitWidth)) {
    APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
    if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;
  }
  break;
}
case ISD::UMIN: {
  // Check if one arg is always less than (or equal) to the other arg.
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
  Known = KnownBits::umin(Known0, Known1);
  if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
    return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
  if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
    return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
  break;
}
case ISD::UMAX: {
  // Check if one arg is always greater than (or equal) to the other arg.
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
  Known = KnownBits::umax(Known0, Known1);
  if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
    return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
  if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
    return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
  break;
}
case ISD::BITREVERSE: {
  SDValue Src = Op.getOperand(0);
  APInt DemandedSrcBits = DemandedBits.reverseBits();
  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
                           Depth + 1))
    return true;
  Known.One = Known2.One.reverseBits();
  Known.Zero = Known2.Zero.reverseBits();
  break;
}
case ISD::BSWAP: {
  SDValue Src = Op.getOperand(0);
  APInt DemandedSrcBits = DemandedBits.byteSwap();
  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
                           Depth + 1))
    return true;
  Known.One = Known2.One.byteSwap();
  Known.Zero = Known2.Zero.byteSwap();
  break;
}
case ISD::CTPOP: {
  // If only 1 bit is demanded, replace with PARITY as long as we're before
  // op legalization.
  // FIXME: Limit to scalars for now.
  if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
                                             Op.getOperand(0)));

  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
  break;
}
case ISD::SIGN_EXTEND_INREG: {
  SDValue Op0 = Op.getOperand(0);
  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  unsigned ExVTBits = ExVT.getScalarSizeInBits();

  // If we only care about the highest bit, don't bother shifting right.
  if (DemandedBits.isSignMask()) {
    unsigned MinSignedBits =
        TLO.DAG.ComputeMinSignedBits(Op0, DemandedElts, Depth + 1);
    bool AlreadySignExtended = ExVTBits >= MinSignedBits;
    // However if the input is already sign extended we expect the sign
    // extension to be dropped altogether later and do not simplify.
    if (!AlreadySignExtended) {
      // Compute the correct shift amount type, which must be getShiftAmountTy
      // for scalar types after legalization.
      EVT ShiftAmtTy = VT;
      if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
        ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

      SDValue ShiftAmt =
          TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
      return TLO.CombineTo(Op,
                           TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
    }
  }

  // If none of the extended bits are demanded, eliminate the sextinreg.
  if (DemandedBits.getActiveBits() <= ExVTBits)
    return TLO.CombineTo(Op, Op0);

  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

  // Since the sign extended bits are demanded, we know that the sign
  // bit is demanded.
  InputDemandedBits.setBit(ExVTBits - 1);

  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1837, __extension__ __PRETTY_FUNCTION__));

  // If the sign bit of the input is known set or clear, then we know the
  // top bits of the result.

  // If the input sign bit is known zero, convert this into a zero extension.
  if (Known.Zero[ExVTBits - 1])
    return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));

  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
    Known.One.setBitsFrom(ExVTBits);
    Known.Zero &= Mask;
  } else { // Input sign bit unknown
    Known.Zero &= Mask;
    Known.One &= Mask;
  }
  break;
}
case ISD::BUILD_PAIR: {
  EVT HalfVT = Op.getOperand(0).getValueType();
  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

  KnownBits KnownLo, KnownHi;

  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
    return true;

  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
    return true;

  Known.Zero = KnownLo.Zero.zext(BitWidth) |
               KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

  Known.One = KnownLo.One.zext(BitWidth) |
              KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
  break;
}
case ISD::ZERO_EXTEND:
case ISD::ZERO_EXTEND_VECTOR_INREG: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned InBits = SrcVT.getScalarSizeInBits();
  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
  bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

  // If none of the top bits are demanded, convert this into an any_extend.
  if (DemandedBits.getActiveBits() <= InBits) {
    // If we only need the non-extended bits of the bottom element
    // then we can just bitcast to the result.
    if (IsVecInReg && DemandedElts == 1 &&
        VT.getSizeInBits() == SrcVT.getSizeInBits() &&
        TLO.DAG.getDataLayout().isLittleEndian())
      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

    unsigned Opc =
        IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
    if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
  }

  APInt InDemandedBits = DemandedBits.trunc(InBits);
  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1906, __extension__ __PRETTY_FUNCTION__));
  assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
 "Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1907, __extension__ __PRETTY_FUNCTION__));
  Known = Known.zext(BitWidth);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
          Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
    return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
  break;
}
case ISD::SIGN_EXTEND:
case ISD::SIGN_EXTEND_VECTOR_INREG: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned InBits = SrcVT.getScalarSizeInBits();
  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
  bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

  // If none of the top bits are demanded, convert this into an any_extend.
  if (DemandedBits.getActiveBits() <= InBits) {
    // If we only need the non-extended bits of the bottom element
    // then we can just bitcast to the result.
    if (IsVecInReg && DemandedElts == 1 &&
        VT.getSizeInBits() == SrcVT.getSizeInBits() &&
        TLO.DAG.getDataLayout().isLittleEndian())
      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

    unsigned Opc =
        IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
    if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
  }

  APInt InDemandedBits = DemandedBits.trunc(InBits);
  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

  // Since some of the sign extended bits are demanded, we know that the sign
  // bit is demanded.
  InDemandedBits.setBit(InBits - 1);

  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1949, __extension__ __PRETTY_FUNCTION__));
  assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
 "Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1950, __extension__ __PRETTY_FUNCTION__));

  // If the sign bit is known one, the top bits match.
  Known = Known.sext(BitWidth);

  // If the sign bit is known zero, convert this to a zero extend.
  if (Known.isNonNegative()) {
    unsigned Opc =
        IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
    if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
  }

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
          Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
    return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
  break;
}
case ISD::ANY_EXTEND:
case ISD::ANY_EXTEND_VECTOR_INREG: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned InBits = SrcVT.getScalarSizeInBits();
  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
  bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

  // If we only need the bottom element then we can just bitcast.
  // TODO: Handle ANY_EXTEND?
  if (IsVecInReg && DemandedElts == 1 &&
      VT.getSizeInBits() == SrcVT.getSizeInBits() &&
      TLO.DAG.getDataLayout().isLittleEndian())
    return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

  APInt InDemandedBits = DemandedBits.trunc(InBits);
  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1989, __extension__ __PRETTY_FUNCTION__));
  assert(Known.getBitWidth() == InBits && "Src width has changed?")(static_cast <bool> (Known.getBitWidth() == InBits &&
 "Src width has changed?") ? void (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1990, __extension__ __PRETTY_FUNCTION__));
  Known = Known.anyext(BitWidth);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
          Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
    return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
  break;
}
case ISD::TRUNCATE: {
  SDValue Src = Op.getOperand(0);

  // Simplify the input, using demanded bit information, and compute the known
  // zero/one bits live out.
  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
  if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  Known = Known.trunc(BitWidth);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
          Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

  // If the input is only used by this truncate, see if we can shrink it based
  // on the known demanded bits.
  if (Src.getNode()->hasOneUse()) {
    switch (Src.getOpcode()) {
    default:
      break;
    case ISD::SRL:
      // Shrink SRL by a constant if none of the high bits shifted in are
      // demanded.
      if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
        // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
        // undesirable.
        break;

      const APInt *ShAmtC =
          TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
      if (!ShAmtC || ShAmtC->uge(BitWidth))
        break;
      uint64_t ShVal = ShAmtC->getZExtValue();

      APInt HighBits =
          APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
      HighBits.lshrInPlace(ShVal);
      HighBits = HighBits.trunc(BitWidth);

      if (!(HighBits & DemandedBits)) {
        // None of the shifted in bits are needed.  Add a truncate of the
        // shift input, then shift it.
        SDValue NewShAmt = TLO.DAG.getConstant(
            ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
        SDValue NewTrunc =
            TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
        return TLO.CombineTo(
            Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
      }
      break;
    }
  }

  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2055, __extension__ __PRETTY_FUNCTION__));
  break;
}
case ISD::AssertZext: {
  // AssertZext demands all of the high bits, plus any of the low bits
  // demanded by its users.
  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
                           TLO, Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")(static_cast <bool> (!Known.hasConflict() && "Bits known to be one AND zero?"
) ? void (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2066, __extension__ __PRETTY_FUNCTION__));

  Known.Zero |= ~InMask;
  break;
}
case ISD::EXTRACT_VECTOR_ELT: {
  SDValue Src = Op.getOperand(0);
  SDValue Idx = Op.getOperand(1);
  ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
  unsigned EltBitWidth = Src.getScalarValueSizeInBits();

  if (SrcEltCnt.isScalable())
    return false;

  // Demand the bits from every vector element without a constant index.
  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
  APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
    if (CIdx->getAPIntValue().ult(NumSrcElts))
      DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
  // anything about the extended bits.
  APInt DemandedSrcBits = DemandedBits;
  if (BitWidth > EltBitWidth)
    DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
                           Depth + 1))
    return true;

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
    if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
            Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
      SDValue NewOp =
          TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  Known = Known2;
  if (BitWidth > EltBitWidth)
    Known = Known.anyext(BitWidth);
  break;
}
case ISD::BITCAST: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

  // If this is an FP->Int bitcast and if the sign bit is the only
  // thing demanded, turn this into a FGETSIGN.
  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
      DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
      SrcVT.isFloatingPoint()) {
    bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
    bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
    if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
        SrcVT != MVT::f128) {
      // Cannot eliminate/lower SHL for f128 yet.
      EVT Ty = OpVTLegal ? VT : MVT::i32;
      // Make a FGETSIGN + SHL to move the sign bit into the appropriate
      // place.  We expect the SHL to be eliminated by other optimizations.
      SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
      unsigned OpVTSizeInBits = Op.getValueSizeInBits();
      if (!OpVTLegal && OpVTSizeInBits > 32)
        Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
      unsigned ShVal = Op.getValueSizeInBits() - 1;
      SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
      return TLO.CombineTo(Op,
                           TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
    }
  }

  // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
  // Demand the elt/bit if any of the original elts/bits are demanded.
  // TODO - bigendian once we have test coverage.
  if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
      TLO.DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = BitWidth / NumSrcEltBits;
    unsigned NumSrcElts = SrcVT.getVectorNumElements();
    APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
    for (unsigned i = 0; i != Scale; ++i) {
      unsigned Offset = i * NumSrcEltBits;
      APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
      if (!Sub.isZero()) {
        DemandedSrcBits |= Sub;
        for (unsigned j = 0; j != NumElts; ++j)
          if (DemandedElts[j])
            DemandedSrcElts.setBit((j * Scale) + i);
      }
    }

    APInt KnownSrcUndef, KnownSrcZero;
    if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
                                   KnownSrcZero, TLO, Depth + 1))
      return true;

    KnownBits KnownSrcBits;
    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
                             KnownSrcBits, TLO, Depth + 1))
      return true;
  } else if ((NumSrcEltBits % BitWidth) == 0 &&
             TLO.DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = NumSrcEltBits / BitWidth;
    unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
    APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
    for (unsigned i = 0; i != NumElts; ++i)
      if (DemandedElts[i]) {
        unsigned Offset = (i % Scale) * BitWidth;
        DemandedSrcBits.insertBits(DemandedBits, Offset);
        DemandedSrcElts.setBit(i / Scale);
      }

    if (SrcVT.isVector()) {
      APInt KnownSrcUndef, KnownSrcZero;
      if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
                                     KnownSrcZero, TLO, Depth + 1))
        return true;
    }

    KnownBits KnownSrcBits;
    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
                             KnownSrcBits, TLO, Depth + 1))
      return true;
  }

  // If this is a bitcast, let computeKnownBits handle it.  Only do this on a
  // recursive call where Known may be useful to the caller.
  if (Depth > 0) {
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false;
  }
  break;
}
case ISD::ADD:
case ISD::MUL:
case ISD::SUB: {
  // Add, Sub, and Mul don't demand any bits in positions beyond that
  // of the highest bit demanded of them.
  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
  SDNodeFlags Flags = Op.getNode()->getFlags();
  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
                           Depth + 1) ||
      SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
                           Depth + 1) ||
      // See if the operation should be performed at a smaller bit width.
      ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
    if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
      // Disable the nsw and nuw flags. We can no longer guarantee that we
      // won't wrap after simplification.
      Flags.setNoSignedWrap(false);
      Flags.setNoUnsignedWrap(false);
      SDValue NewOp =
          TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
      return TLO.CombineTo(Op, NewOp);
    }
    return true;
  }

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Flags.setNoSignedWrap(false);
      Flags.setNoUnsignedWrap(false);
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp =
          TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If we have a constant operand, we may be able to turn it into -1 if we
  // do not demand the high bits. This can make the constant smaller to
  // encode, allow more general folding, or match specialized instruction
  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
  // is probably not useful (and could be detrimental).
  ConstantSDNode *C = isConstOrConstSplat(Op1);
  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
  if (C && !C->isAllOnes() && !C->isOne() &&
      (C->getAPIntValue() | HighMask).isAllOnes()) {
    SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
    // Disable the nsw and nuw flags. We can no longer guarantee that we
    // won't wrap after simplification.
    Flags.setNoSignedWrap(false);
    Flags.setNoUnsignedWrap(false);
    SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
    return TLO.CombineTo(Op, NewOp);
  }

  LLVM_FALLTHROUGH[[gnu::fallthrough]];
}
default:
  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
    if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
                                          Known, TLO, Depth))
      return true;
    break;
  }

  // Just use computeKnownBits to compute output bits.
  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
  break;
}

// If we know the value of all of the demanded bits, return this as a
// constant.
if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
  // Avoid folding to a constant if any OpaqueConstant is involved.
  const SDNode *N = Op.getNode();
  for (SDNode *Op :
       llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
      if (C->isOpaque())
        return false;
  }
  if (VT.isInteger())
    return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
  if (VT.isFloatingPoint())
    return TLO.CombineTo(
        Op,
        TLO.DAG.getConstantFP(
            APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
}

return false;
2302}

2304bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
                                              const APInt &DemandedElts,
                                              APInt &KnownUndef,
                                              APInt &KnownZero,
                                              DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                      !DCI.isBeforeLegalizeOps());

bool Simplified =
    SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
if (Simplified) {
  DCI.AddToWorklist(Op.getNode());
  DCI.CommitTargetLoweringOpt(TLO);
}

return Simplified;
2321}

2323/// Given a vector binary operation and known undefined elements for each input
2324/// operand, compute whether each element of the output is undefined.
2325static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
                                       const APInt &UndefOp0,
                                       const APInt &UndefOp1) {
EVT VT = BO.getValueType();
assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2330, __extension__ __PRETTY_FUNCTION__))
       "Vector binop only")(static_cast <bool> (DAG.getTargetLoweringInfo().isBinOp
(BO.getOpcode()) && VT.isVector() && "Vector binop only"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2330, __extension__ __PRETTY_FUNCTION__));

EVT EltVT = VT.getVectorElementType();
unsigned NumElts = VT.getVectorNumElements();
assert(UndefOp0.getBitWidth() == NumElts &&(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2335, __extension__ __PRETTY_FUNCTION__))
       UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")(static_cast <bool> (UndefOp0.getBitWidth() == NumElts &&
 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"
) ? void (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2335, __extension__ __PRETTY_FUNCTION__));

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
                                 const APInt &UndefVals) {
  if (UndefVals[Index])
    return DAG.getUNDEF(EltVT);

  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
    // Try hard to make sure that the getNode() call is not creating temporary
    // nodes. Ignore opaque integers because they do not constant fold.
    SDValue Elt = BV->getOperand(Index);
    auto *C = dyn_cast<ConstantSDNode>(Elt);
    if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
      return Elt;
  }

  return SDValue();
};

APInt KnownUndef = APInt::getZero(NumElts);
for (unsigned i = 0; i != NumElts; ++i) {
  // If both inputs for this element are either constant or undef and match
  // the element type, compute the constant/undef result for this element of
  // the vector.
  // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
  // not handle FP constants. The code within getNode() should be refactored
  // to avoid the danger of creating a bogus temporary node here.
  SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
  SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
  if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
    if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
      KnownUndef.setBit(i);
}
return KnownUndef;
2369}

2371bool TargetLowering::SimplifyDemandedVectorElts(
  SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
  bool AssumeSingleUse) const {
EVT VT = Op.getValueType();
unsigned Opcode = Op.getOpcode();
APInt DemandedElts = OriginalDemandedElts;
unsigned NumElts = DemandedElts.getBitWidth();
assert(VT.isVector() && "Expected vector op")(static_cast <bool> (VT.isVector() && "Expected vector op"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2379, __extension__ __PRETTY_FUNCTION__));

KnownUndef = KnownZero = APInt::getZero(NumElts);

// TODO: For now we assume we know nothing about scalable vectors.
if (VT.isScalableVector())
  return false;

assert(VT.getVectorNumElements() == NumElts &&(static_cast <bool> (VT.getVectorNumElements() == NumElts
 && "Mask size mismatches value type element count!")
 ? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2388, __extension__ __PRETTY_FUNCTION__))
       "Mask size mismatches value type element count!")(static_cast <bool> (VT.getVectorNumElements() == NumElts
 && "Mask size mismatches value type element count!")
 ? void (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2388, __extension__ __PRETTY_FUNCTION__));

// Undef operand.
if (Op.isUndef()) {
  KnownUndef.setAllBits();
  return false;
}

// If Op has other users, assume that all elements are needed.
if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
  DemandedElts.setAllBits();

// Not demanding any elements from Op.
if (DemandedElts == 0) {
  KnownUndef.setAllBits();
  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
}

// Limit search depth.
if (Depth >= SelectionDAG::MaxRecursionDepth)
  return false;

SDLoc DL(Op);
unsigned EltSizeInBits = VT.getScalarSizeInBits();

// Helper for demanding the specified elements and all the bits of both binary
// operands.
auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
  SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
                                                         TLO.DAG, Depth + 1);
  SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
                                                         TLO.DAG, Depth + 1);
  if (NewOp0 || NewOp1) {
    SDValue NewOp = TLO.DAG.getNode(
        Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
    return TLO.CombineTo(Op, NewOp);
  }
  return false;
};

switch (Opcode) {
case ISD::SCALAR_TO_VECTOR: {
  if (!DemandedElts[0]) {
    KnownUndef.setAllBits();
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
  }
  SDValue ScalarSrc = Op.getOperand(0);
  if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
    SDValue Src = ScalarSrc.getOperand(0);
    SDValue Idx = ScalarSrc.getOperand(1);
    EVT SrcVT = Src.getValueType();

    ElementCount SrcEltCnt = SrcVT.getVectorElementCount();

    if (SrcEltCnt.isScalable())
      return false;

    unsigned NumSrcElts = SrcEltCnt.getFixedValue();
    if (isNullConstant(Idx)) {
      APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
      APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
      APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
      if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                     TLO, Depth + 1))
        return true;
    }
  }
  KnownUndef.setHighBits(NumElts - 1);
  break;
}
case ISD::BITCAST: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();

  // We only handle vectors here.
  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
  if (!SrcVT.isVector())
    break;

  // Fast handling of 'identity' bitcasts.
  unsigned NumSrcElts = SrcVT.getVectorNumElements();
  if (NumSrcElts == NumElts)
    return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
                                      KnownZero, TLO, Depth + 1);

  APInt SrcDemandedElts, SrcZero, SrcUndef;

  // Bitcast from 'large element' src vector to 'small element' vector, we
  // must demand a source element if any DemandedElt maps to it.
  if ((NumElts % NumSrcElts) == 0) {
    unsigned Scale = NumElts / NumSrcElts;
    SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
    if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                   TLO, Depth + 1))
      return true;

    // Try calling SimplifyDemandedBits, converting demanded elts to the bits
    // of the large element.
    // TODO - bigendian once we have test coverage.
    if (TLO.DAG.getDataLayout().isLittleEndian()) {
      unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
      APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
      for (unsigned i = 0; i != NumElts; ++i)
        if (DemandedElts[i]) {
          unsigned Ofs = (i % Scale) * EltSizeInBits;
          SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
        }

      KnownBits Known;
      if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
                               TLO, Depth + 1))
        return true;
    }

    // If the src element is zero/undef then all the output elements will be -
    // only demanded elements are guaranteed to be correct.
    for (unsigned i = 0; i != NumSrcElts; ++i) {
      if (SrcDemandedElts[i]) {
        if (SrcZero[i])
          KnownZero.setBits(i * Scale, (i + 1) * Scale);
        if (SrcUndef[i])
          KnownUndef.setBits(i * Scale, (i + 1) * Scale);
      }
    }
  }

  // Bitcast from 'small element' src vector to 'large element' vector, we
  // demand all smaller source elements covered by the larger demanded element
  // of this vector.
  if ((NumSrcElts % NumElts) == 0) {
    unsigned Scale = NumSrcElts / NumElts;
    SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
    if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                   TLO, Depth + 1))
      return true;

    // If all the src elements covering an output element are zero/undef, then
    // the output element will be as well, assuming it was demanded.
    for (unsigned i = 0; i != NumElts; ++i) {
      if (DemandedElts[i]) {
        if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
          KnownZero.setBit(i);
        if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
          KnownUndef.setBit(i);
      }
    }
  }
  break;
}
case ISD::BUILD_VECTOR: {
  // Check all elements and simplify any unused elements with UNDEF.
  if (!DemandedElts.isAllOnes()) {
    // Don't simplify BROADCASTS.
    if (llvm::any_of(Op->op_values(),
                     [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
      SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
      bool Updated = false;
      for (unsigned i = 0; i != NumElts; ++i) {
        if (!DemandedElts[i] && !Ops[i].isUndef()) {
          Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
          KnownUndef.setBit(i);
          Updated = true;
        }
      }
      if (Updated)
        return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
    }
  }
  for (unsigned i = 0; i != NumElts; ++i) {
    SDValue SrcOp = Op.getOperand(i);
    if (SrcOp.isUndef()) {
      KnownUndef.setBit(i);
    } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
               (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
      KnownZero.setBit(i);
    }
  }
  break;
}
case ISD::CONCAT_VECTORS: {
  EVT SubVT = Op.getOperand(0).getValueType();
  unsigned NumSubVecs = Op.getNumOperands();
  unsigned NumSubElts = SubVT.getVectorNumElements();
  for (unsigned i = 0; i != NumSubVecs; ++i) {
    SDValue SubOp = Op.getOperand(i);
    APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
    APInt SubUndef, SubZero;
    if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
                                   Depth + 1))
      return true;
    KnownUndef.insertBits(SubUndef, i * NumSubElts);
    KnownZero.insertBits(SubZero, i * NumSubElts);
  }
  break;
}
case ISD::INSERT_SUBVECTOR: {
  // Demand any elements from the subvector and the remainder from the src its
  // inserted into.
  SDValue Src = Op.getOperand(0);
  SDValue Sub = Op.getOperand(1);
  uint64_t Idx = Op.getConstantOperandVal(2);
  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
  APInt DemandedSrcElts = DemandedElts;
  DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);

  APInt SubUndef, SubZero;
  if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
                                 Depth + 1))
    return true;

  // If none of the src operand elements are demanded, replace it with undef.
  if (!DemandedSrcElts && !Src.isUndef())
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
                                             TLO.DAG.getUNDEF(VT), Sub,
                                             Op.getOperand(2)));

  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
                                 TLO, Depth + 1))
    return true;
  KnownUndef.insertBits(SubUndef, Idx);
  KnownZero.insertBits(SubZero, Idx);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
    SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
        Src, DemandedSrcElts, TLO.DAG, Depth + 1);
    SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
        Sub, DemandedSubElts, TLO.DAG, Depth + 1);
    if (NewSrc || NewSub) {
      NewSrc = NewSrc ? NewSrc : Src;
      NewSub = NewSub ? NewSub : Sub;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
                                      NewSub, Op.getOperand(2));
      return TLO.CombineTo(Op, NewOp);
    }
  }
  break;
}
case ISD::EXTRACT_SUBVECTOR: {
  // Offset the demanded elts by the subvector index.
  SDValue Src = Op.getOperand(0);
  if (Src.getValueType().isScalableVector())
    break;
  uint64_t Idx = Op.getConstantOperandVal(1);
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);

  APInt SrcUndef, SrcZero;
  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
                                 Depth + 1))
    return true;
  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
  KnownZero = SrcZero.extractBits(NumElts, Idx);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedElts.isAllOnes()) {
    SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
        Src, DemandedSrcElts, TLO.DAG, Depth + 1);
    if (NewSrc) {
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
                                      Op.getOperand(1));
      return TLO.CombineTo(Op, NewOp);
    }
  }
  break;
}
case ISD::INSERT_VECTOR_ELT: {
  SDValue Vec = Op.getOperand(0);
  SDValue Scl = Op.getOperand(1);
  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

  // For a legal, constant insertion index, if we don't need this insertion
  // then strip it, else remove it from the demanded elts.
  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
    unsigned Idx = CIdx->getZExtValue();
    if (!DemandedElts[Idx])
      return TLO.CombineTo(Op, Vec);

    APInt DemandedVecElts(DemandedElts);
    DemandedVecElts.clearBit(Idx);
    if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
                                   KnownZero, TLO, Depth + 1))
      return true;

    KnownUndef.setBitVal(Idx, Scl.isUndef());

    KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
    break;
  }

  APInt VecUndef, VecZero;
  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
                                 Depth + 1))
    return true;
  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
  break;
}
case ISD::VSELECT: {
  // Try to transform the select condition based on the current demanded
  // elements.
  // TODO: If a condition element is undef, we can choose from one arm of the
  //       select (and if one arm is undef, then we can propagate that to the
  //       result).
  // TODO - add support for constant vselect masks (see IR version of this).
  APInt UnusedUndef, UnusedZero;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
                                 UnusedZero, TLO, Depth + 1))
    return true;

  // See if we can simplify either vselect operand.
  APInt DemandedLHS(DemandedElts);
  APInt DemandedRHS(DemandedElts);
  APInt UndefLHS, ZeroLHS;
  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
                                 ZeroLHS, TLO, Depth + 1))
    return true;
  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
                                 ZeroRHS, TLO, Depth + 1))
    return true;

  KnownUndef = UndefLHS & UndefRHS;
  KnownZero = ZeroLHS & ZeroRHS;
  break;
}
case ISD::VECTOR_SHUFFLE: {
  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

  // Collect demanded elements from shuffle operands..
  APInt DemandedLHS(NumElts, 0);
  APInt DemandedRHS(NumElts, 0);
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = ShuffleMask[i];
    if (M < 0 || !DemandedElts[i])
      continue;
    assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")(static_cast <bool> (0 <= M && M < (int)(
* NumElts) && "Shuffle index out of range") ? void (
0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2724, __extension__ __PRETTY_FUNCTION__));
    if (M < (int)NumElts)
      DemandedLHS.setBit(M);
    else
      DemandedRHS.setBit(M - NumElts);
  }

  // See if we can simplify either shuffle operand.
  APInt UndefLHS, ZeroLHS;
  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
                                 ZeroLHS, TLO, Depth + 1))
    return true;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
                                 ZeroRHS, TLO, Depth + 1))
    return true;

  // Simplify mask using undef elements from LHS/RHS.
  bool Updated = false;
  bool IdentityLHS = true, IdentityRHS = true;
  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
  for (unsigned i = 0; i != NumElts; ++i) {
    int &M = NewMask[i];
    if (M < 0)
      continue;
    if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
        (M >= (int)NumElts && UndefRHS[M - NumElts])) {
      Updated = true;
      M = -1;
    }
    IdentityLHS &= (M < 0) || (M == (int)i);
    IdentityRHS &= (M < 0) || ((M - NumElts) == i);
  }

  // Update legal shuffle masks based on demanded elements if it won't reduce
  // to Identity which can cause premature removal of the shuffle mask.
  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
    SDValue LegalShuffle =
        buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
                                NewMask, TLO.DAG);
    if (LegalShuffle)
      return TLO.CombineTo(Op, LegalShuffle);
  }

  // Propagate undef/zero elements from LHS/RHS.
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = ShuffleMask[i];
    if (M < 0) {
      KnownUndef.setBit(i);
    } else if (M < (int)NumElts) {
      if (UndefLHS[M])
        KnownUndef.setBit(i);
      if (ZeroLHS[M])
        KnownZero.setBit(i);
    } else {
      if (UndefRHS[M - NumElts])
        KnownUndef.setBit(i);
      if (ZeroRHS[M - NumElts])
        KnownZero.setBit(i);
    }
  }
  break;
}
case ISD::ANY_EXTEND_VECTOR_INREG:
case ISD::SIGN_EXTEND_VECTOR_INREG:
case ISD::ZERO_EXTEND_VECTOR_INREG: {
  APInt SrcUndef, SrcZero;
  SDValue Src = Op.getOperand(0);
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
                                 Depth + 1))
    return true;
  KnownZero = SrcZero.zextOrTrunc(NumElts);
  KnownUndef = SrcUndef.zextOrTrunc(NumElts);

  if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
      Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
      DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
    // aext - if we just need the bottom element then we can bitcast.
    return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
  }

  if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
    // zext(undef) upper bits are guaranteed to be zero.
    if (DemandedElts.isSubsetOf(KnownUndef))
      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
    KnownUndef.clearAllBits();
  }
  break;
}

// TODO: There are more binop opcodes that could be handled here - MIN,
// MAX, saturated math, etc.
case ISD::OR:
case ISD::XOR:
case ISD::ADD:
case ISD::SUB:
case ISD::FADD:
case ISD::FSUB:
case ISD::FMUL:
case ISD::FDIV:
case ISD::FREM: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
                                 Depth + 1))
    return true;
  APInt UndefLHS, ZeroLHS;
  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
                                 Depth + 1))
    return true;

  KnownZero = ZeroLHS & ZeroRHS;
  KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  // TODO - use KnownUndef to relax the demandedelts?
  if (!DemandedElts.isAllOnes())
    if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
      return true;
  break;
}
case ISD::SHL:
case ISD::SRL:
case ISD::SRA:
case ISD::ROTL:
case ISD::ROTR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
                                 Depth + 1))
    return true;
  APInt UndefLHS, ZeroLHS;
  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
                                 Depth + 1))
    return true;

  KnownZero = ZeroLHS;
  KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?

  // Attempt to avoid multi-use ops if we don't need anything from them.
  // TODO - use KnownUndef to relax the demandedelts?
  if (!DemandedElts.isAllOnes())
    if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
      return true;
  break;
}
case ISD::MUL:
case ISD::AND: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  APInt SrcUndef, SrcZero;
  if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
                                 Depth + 1))
    return true;
  if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
                                 TLO, Depth + 1))
    return true;

  // If either side has a zero element, then the result element is zero, even
  // if the other is an UNDEF.
  // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
  // and then handle 'and' nodes with the rest of the binop opcodes.
  KnownZero |= SrcZero;
  KnownUndef &= SrcUndef;
  KnownUndef &= ~KnownZero;

  // Attempt to avoid multi-use ops if we don't need anything from them.
  // TODO - use KnownUndef to relax the demandedelts?
  if (!DemandedElts.isAllOnes())
    if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
      return true;
  break;
}
case ISD::TRUNCATE:
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
                                 KnownZero, TLO, Depth + 1))
    return true;

  if (Op.getOpcode() == ISD::ZERO_EXTEND) {
    // zext(undef) upper bits are guaranteed to be zero.
    if (DemandedElts.isSubsetOf(KnownUndef))
      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
    KnownUndef.clearAllBits();
  }
  break;
default: {
  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
    if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
                                                KnownZero, TLO, Depth))
      return true;
  } else {
    KnownBits Known;
    APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
    if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
                             TLO, Depth, AssumeSingleUse))
      return true;
  }
  break;
}
}
assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(static_cast <bool> ((KnownUndef & KnownZero) == 0 &&
 "Elements flagged as undef AND zero") ? void (0) : __assert_fail
 ("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2933, __extension__ __PRETTY_FUNCTION__));

// Constant fold all undef cases.
// TODO: Handle zero cases as well.
if (DemandedElts.isSubsetOf(KnownUndef))
  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

return false;
2941}

2943/// Determine which of the bits specified in Mask are known to be either zero or
2944/// one and return them in the Known.
2945void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
                                                 KnownBits &Known,
                                                 const APInt &DemandedElts,
                                                 const SelectionDAG &DAG,
                                                 unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
       "Should use MaskedValueIsZero if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__))
       " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2955, __extension__ __PRETTY_FUNCTION__));
Known.resetAll();
2957}

2959void TargetLowering::computeKnownBitsForTargetInstr(
  GISelKnownBits &Analysis, Register R, KnownBits &Known,
  const APInt &DemandedElts, const MachineRegisterInfo &MRI,
  unsigned Depth) const {
Known.resetAll();
2964}

2966void TargetLowering::computeKnownBitsForFrameIndex(
const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
// The low bits are known zero if the pointer is aligned.
Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2970}

2972Align TargetLowering::computeKnownAlignForTargetInstr(
GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
unsigned Depth) const {
return Align(1);
2976}

2978/// This method can be implemented by targets that want to expose additional
2979/// information about sign bits to the DAG Combiner.
2980unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
                                                       const APInt &,
                                                       const SelectionDAG &,
                                                       unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
       "Should use ComputeNumSignBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__))
       " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2989, __extension__ __PRETTY_FUNCTION__));
return 1;
2991}

2993unsigned TargetLowering::computeNumSignBitsForTargetInstr(
GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
const MachineRegisterInfo &MRI, unsigned Depth) const {
return 1;
2997}

2999bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
  TargetLoweringOpt &TLO, unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
       "Should use SimplifyDemandedVectorElts if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__))
       " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3007, __extension__ __PRETTY_FUNCTION__));
return false;
3009}

3011bool TargetLowering::SimplifyDemandedBitsForTargetNode(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
       "Should use SimplifyDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__))
       " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3019, __extension__ __PRETTY_FUNCTION__));
computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
return false;
3022}

3024SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  SelectionDAG &DAG, unsigned Depth) const {
assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
    (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
    "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__))
    " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3033, __extension__ __PRETTY_FUNCTION__));
return SDValue();
3035}

3037SDValue
3038TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
                                      SDValue N1, MutableArrayRef<int> Mask,
                                      SelectionDAG &DAG) const {
bool LegalMask = isShuffleMaskLegal(Mask, VT);
if (!LegalMask) {
  std::swap(N0, N1);
  ShuffleVectorSDNode::commuteMask(Mask);
  LegalMask = isShuffleMaskLegal(Mask, VT);
}

if (!LegalMask)
  return SDValue();

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3052}

3054const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
return nullptr;
3056}

3058bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
  bool PoisonOnly, unsigned Depth) const {
assert((static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
    (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
    "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__))
    " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3067, __extension__ __PRETTY_FUNCTION__));
return false;
3069}

3071bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
                                                const SelectionDAG &DAG,
                                                bool SNaN,
                                                unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
       "Should use isKnownNeverNaN if you don't know whether Op"(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__))
       " is a target node!")(static_cast <bool> ((Op.getOpcode() >= ISD::BUILTIN_OP_END
 || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode
() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID
) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? void (0) : __assert_fail ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3080, __extension__ __PRETTY_FUNCTION__));
return false;
3082}

3084// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3085// work with truncating build vectors and vectors with elements of less than
3086// 8 bits.
3087bool TargetLowering::isConstTrueVal(const SDNode *N) const {
if (!N)
  return false;

APInt CVal;
if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
  CVal = CN->getAPIntValue();
} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
  auto *CN = BV->getConstantSplatNode();
  if (!CN)
    return false;

  // If this is a truncating build vector, truncate the splat value.
  // Otherwise, we may fail to match the expected values below.
  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
  CVal = CN->getAPIntValue();
  if (BVEltWidth < CVal.getBitWidth())
    CVal = CVal.trunc(BVEltWidth);
} else {
  return false;
}

switch (getBooleanContents(N->getValueType(0))) {
case UndefinedBooleanContent:
  return CVal[0];
case ZeroOrOneBooleanContent:
  return CVal.isOne();
case ZeroOrNegativeOneBooleanContent:
  return CVal.isAllOnes();
}

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
 "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3118);
3119}

3121bool TargetLowering::isConstFalseVal(const SDNode *N) const {
if (!N)
  return false;

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
if (!CN) {
  const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
  if (!BV)
    return false;

  // Only interested in constant splats, we don't care about undef
  // elements in identifying boolean constants and getConstantSplatNode
  // returns NULL if all ops are undef;
  CN = BV->getConstantSplatNode();
  if (!CN)
    return false;
}

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
  return !CN->getAPIntValue()[0];

return CN->isZero();
3143}

3145bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
                                     bool SExt) const {
if (VT == MVT::i1)
  return N->isOne();

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
switch (Cnt) {
case TargetLowering::ZeroOrOneBooleanContent:
  // An extended value of 1 is always true, unless its original type is i1,
  // in which case it will be sign extended to -1.
  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
case TargetLowering::UndefinedBooleanContent:
case TargetLowering::ZeroOrNegativeOneBooleanContent:
  return N->isAllOnes() && SExt;
}
llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3160);
3161}

3163/// This helper function of SimplifySetCC tries to optimize the comparison when
3164/// either operand of the SetCC node is a bitwise-and instruction.
3165SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
                                       ISD::CondCode Cond, const SDLoc &DL,
                                       DAGCombinerInfo &DCI) const {
// Match these patterns in any of their permutations:
// (X & Y) == Y
// (X & Y) != Y
if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
  std::swap(N0, N1);

EVT OpVT = N0.getValueType();
if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
    (Cond != ISD::SETEQ && Cond != ISD::SETNE))
  return SDValue();

SDValue X, Y;
if (N0.getOperand(0) == N1) {
  X = N0.getOperand(1);
  Y = N0.getOperand(0);
} else if (N0.getOperand(1) == N1) {
  X = N0.getOperand(0);
  Y = N0.getOperand(1);
} else {
  return SDValue();
}

SelectionDAG &DAG = DCI.DAG;
SDValue Zero = DAG.getConstant(0, DL, OpVT);
if (DAG.isKnownToBeAPowerOfTwo(Y)) {
  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
  // Note that where Y is variable and is known to have at most one bit set
  // (for example, if it is Z & 1) we cannot do this; the expressions are not
  // equivalent when Y == 0.
  assert(OpVT.isInteger())(static_cast <bool> (OpVT.isInteger()) ? void (0) : __assert_fail
 ("OpVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3197, __extension__ __PRETTY_FUNCTION__));
  Cond = ISD::getSetCCInverse(Cond, OpVT);
  if (DCI.isBeforeLegalizeOps() ||
      isCondCodeLegal(Cond, N0.getSimpleValueType()))
    return DAG.getSetCC(DL, VT, N0, Zero, Cond);
} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
  // If the target supports an 'and-not' or 'and-complement' logic operation,
  // try to use that to make a comparison operation more efficient.
  // But don't do this transform if the mask is a single bit because there are
  // more efficient ways to deal with that case (for example, 'bt' on x86 or
  // 'rlwinm' on PPC).

  // Bail out if the compare operand that we want to turn into a zero is
  // already a zero (otherwise, infinite loop).
  auto *YConst = dyn_cast<ConstantSDNode>(Y);
  if (YConst && YConst->isZero())
    return SDValue();

  // Transform this into: ~X & Y == 0.
  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
}

return SDValue();
3222}

3224/// There are multiple IR patterns that could be checking whether certain
3225/// truncation of a signed number would be lossy or not. The pattern which is
3226/// best at IR level, may not lower optimally. Thus, we want to unfold it.
3227/// We are looking for the following pattern: (KeptBits is a constant)
3228///   (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3229/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3230/// KeptBits also can't be 1, that would have been folded to  %x dstcond 0
3231/// We will unfold it into the natural trunc+sext pattern:
3232///   ((%x << C) a>> C) dstcond %x
3233/// Where  C = bitwidth(x) - KeptBits  and  C u< bitwidth(x)
3234SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
  const SDLoc &DL) const {
// We must be comparing with a constant.
ConstantSDNode *C1;
if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
  return SDValue();

// N0 should be:  add %x, (1 << (KeptBits-1))
if (N0->getOpcode() != ISD::ADD)
  return SDValue();

// And we must be 'add'ing a constant.
ConstantSDNode *C01;
if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
  return SDValue();

SDValue X = N0->getOperand(0);
EVT XVT = X.getValueType();

// Validate constants ...

APInt I1 = C1->getAPIntValue();

ISD::CondCode NewCond;
if (Cond == ISD::CondCode::SETULT) {
  NewCond = ISD::CondCode::SETEQ;
} else if (Cond == ISD::CondCode::SETULE) {
  NewCond = ISD::CondCode::SETEQ;
  // But need to 'canonicalize' the constant.
  I1 += 1;
} else if (Cond == ISD::CondCode::SETUGT) {
  NewCond = ISD::CondCode::SETNE;
  // But need to 'canonicalize' the constant.
  I1 += 1;
} else if (Cond == ISD::CondCode::SETUGE) {
  NewCond = ISD::CondCode::SETNE;
} else
  return SDValue();

APInt I01 = C01->getAPIntValue();

auto checkConstants = [&I1, &I01]() -> bool {
  // Both of them must be power-of-two, and the constant from setcc is bigger.
  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
};

if (checkConstants()) {
  // Great, e.g. got  icmp ult i16 (add i16 %x, 128), 256
} else {
  // What if we invert constants? (and the target predicate)
  I1.negate();
  I01.negate();
  assert(XVT.isInteger())(static_cast <bool> (XVT.isInteger()) ? void (0) : __assert_fail
 ("XVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3287, __extension__ __PRETTY_FUNCTION__));
  NewCond = getSetCCInverse(NewCond, XVT);
  if (!checkConstants())
    return SDValue();
  // Great, e.g. got  icmp uge i16 (add i16 %x, -128), -256
}

// They are power-of-two, so which bit is set?
const unsigned KeptBits = I1.logBase2();
const unsigned KeptBitsMinusOne = I01.logBase2();

// Magic!
if (KeptBits != (KeptBitsMinusOne + 1))
  return SDValue();
assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (KeptBits > 0 && KeptBits
 < XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3301, __extension__ __PRETTY_FUNCTION__));

// We don't want to do this in every single case.
SelectionDAG &DAG = DCI.DAG;
if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
        XVT, KeptBits))
  return SDValue();

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")(static_cast <bool> (MaskedBits > 0 && MaskedBits
 < XVT.getSizeInBits() && "unreachable") ? void (0
) : __assert_fail ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3310, __extension__ __PRETTY_FUNCTION__));

// Unfold into:  ((%x << C) a>> C) cond %x
// Where 'cond' will be either 'eq' or 'ne'.
SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

return T2;
3320}

3322// (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
3323SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
  EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
  DAGCombinerInfo &DCI, const SDLoc &DL) const {
assert(isConstOrConstSplat(N1C) &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
 isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
 "Should be a comparison with 0.") ? void (0) : __assert_fail
 ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))
       isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&(static_cast <bool> (isConstOrConstSplat(N1C) &&
 isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
 "Should be a comparison with 0.") ? void (0) : __assert_fail
 ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__))
       "Should be a comparison with 0.")(static_cast <bool> (isConstOrConstSplat(N1C) &&
 isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
 "Should be a comparison with 0.") ? void (0) : __assert_fail
 ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isZero() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3328, __extension__ __PRETTY_FUNCTION__));
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3330, __extension__ __PRETTY_FUNCTION__))
       "Valid only for [in]equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Valid only for [in]equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3330, __extension__ __PRETTY_FUNCTION__));

unsigned NewShiftOpcode;
SDValue X, C, Y;

SelectionDAG &DAG = DCI.DAG;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

// Look for '(C l>>/<< Y)'.
auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
  // The shift should be one-use.
  if (!V.hasOneUse())
    return false;
  unsigned OldShiftOpcode = V.getOpcode();
  switch (OldShiftOpcode) {
  case ISD::SHL:
    NewShiftOpcode = ISD::SRL;
    break;
  case ISD::SRL:
    NewShiftOpcode = ISD::SHL;
    break;
  default:
    return false; // must be a logical shift.
  }
  // We should be shifting a constant.
  // FIXME: best to use isConstantOrConstantVector().
  C = V.getOperand(0);
  ConstantSDNode *CC =
      isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
  if (!CC)
    return false;
  Y = V.getOperand(1);

  ConstantSDNode *XC =
      isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
  return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
      X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
};

// LHS of comparison should be an one-use 'and'.
if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
  return SDValue();

X = N0.getOperand(0);
SDValue Mask = N0.getOperand(1);

// 'and' is commutative!
if (!Match(Mask)) {
  std::swap(X, Mask);
  if (!Match(Mask))
    return SDValue();
}

EVT VT = X.getValueType();

// Produce:
// ((X 'OppositeShiftOpcode' Y) & C) Cond 0
SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
return T2;
3391}

3393/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3394/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3395/// handle the commuted versions of these patterns.
3396SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
                                         ISD::CondCode Cond, const SDLoc &DL,
                                         DAGCombinerInfo &DCI) const {
unsigned BOpcode = N0.getOpcode();
assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
 ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__))
       "Unexpected binop")(static_cast <bool> ((BOpcode == ISD::ADD || BOpcode ==
 ISD::SUB || BOpcode == ISD::XOR) && "Unexpected binop"
) ? void (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__));
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Unexpected condcode") ? void (0) : __assert_fail
 ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3402, __extension__ __PRETTY_FUNCTION__));

// (X + Y) == X --> Y == 0
// (X - Y) == X --> Y == 0
// (X ^ Y) == X --> Y == 0
SelectionDAG &DAG = DCI.DAG;
EVT OpVT = N0.getValueType();
SDValue X = N0.getOperand(0);
SDValue Y = N0.getOperand(1);
if (X == N1)
  return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

if (Y != N1)
  return SDValue();

// (X + Y) == Y --> X == 0
// (X ^ Y) == Y --> X == 0
if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
  return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

// The shift would not be valid if the operands are boolean (i1).
if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
  return SDValue();

// (X - Y) == Y --> X == Y << 1
EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
                               !DCI.isBeforeLegalize());
SDValue One = DAG.getConstant(1, DL, ShiftVT);
SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
if (!DCI.isCalledByLegalizer())
  DCI.AddToWorklist(YShl1.getNode());
return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3434}

3436static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
                                    SDValue N0, const APInt &C1,
                                    ISD::CondCode Cond, const SDLoc &dl,
                                    SelectionDAG &DAG) {
// Look through truncs that don't change the value of a ctpop.
// FIXME: Add vector support? Need to be careful with setcc result type below.
SDValue CTPOP = N0;
if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
    N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
  CTPOP = N0.getOperand(0);

if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
  return SDValue();

EVT CTVT = CTPOP.getValueType();
SDValue CTOp = CTPOP.getOperand(0);

// If this is a vector CTPOP, keep the CTPOP if it is legal.
// TODO: Should we check if CTPOP is legal(or custom) for scalars?
if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
  return SDValue();

// (ctpop x) u< 2 -> (x & x-1) == 0
// (ctpop x) u> 1 -> (x & x-1) != 0
if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
  unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
  if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
    return SDValue();
  if (C1 == 0 && (Cond == ISD::SETULT))
    return SDValue(); // This is handled elsewhere.

  unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);

  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
  SDValue Result = CTOp;
  for (unsigned i = 0; i < Passes; i++) {
    SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
    Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
  }
  ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
  return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
}

// If ctpop is not supported, expand a power-of-2 comparison based on it.
if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
  // For scalars, keep CTPOP if it is legal or custom.
  if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
    return SDValue();
  // This is based on X86's custom lowering for CTPOP which produces more
  // instructions than the expansion here.

  // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
  // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
  SDValue Zero = DAG.getConstant(0, dl, CTVT);
  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
  assert(CTVT.isInteger())(static_cast <bool> (CTVT.isInteger()) ? void (0) : __assert_fail
 ("CTVT.isInteger()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3491, __extension__ __PRETTY_FUNCTION__));
  ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
  SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
  SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
  unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
  return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
}

return SDValue();
3502}

3504/// Try to simplify a setcc built with the specified operands and cc. If it is
3505/// unable to simplify it, return a null SDValue.
3506SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
                                    ISD::CondCode Cond, bool foldBooleans,
                                    DAGCombinerInfo &DCI,
                                    const SDLoc &dl) const {
SelectionDAG &DAG = DCI.DAG;
const DataLayout &Layout = DAG.getDataLayout();
EVT OpVT = N0.getValueType();

// Constant fold or commute setcc.
if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
  return Fold;

// Ensure that the constant occurs on the RHS and fold constant comparisons.
// TODO: Handle non-splat vector constants. All undef causes trouble.
// FIXME: We can't yet fold constant scalable vector splats, so avoid an
// infinite loop here when we encounter one.
ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
if (isConstOrConstSplat(N0) &&
    (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
    (DCI.isBeforeLegalizeOps() ||
     isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

// If we have a subtract with the same 2 non-constant operands as this setcc
// -- but in reverse order -- then try to commute the operands of this setcc
// to match. A matching pair of setcc (cmp) and sub may be combined into 1
// instruction on some targets.
if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
    (DCI.isBeforeLegalizeOps() ||
     isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
    DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
    !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

if (auto *N1C = isConstOrConstSplat(N1)) {
  const APInt &C1 = N1C->getAPIntValue();

  // Optimize some CTPOP cases.
  if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
    return V;

  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
  // equality comparison, then we're just comparing whether X itself is
  // zero.
  if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
      N0.getOperand(0).getOpcode() == ISD::CTLZ &&
      isPowerOf2_32(N0.getScalarValueSizeInBits())) {
    if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
      if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
          ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
        if ((C1 == 0) == (Cond == ISD::SETEQ)) {
          // (srl (ctlz x), 5) == 0  -> X != 0
          // (srl (ctlz x), 5) != 1  -> X != 0
          Cond = ISD::SETNE;
        } else {
          // (srl (ctlz x), 5) != 0  -> X == 0
          // (srl (ctlz x), 5) == 1  -> X == 0
          Cond = ISD::SETEQ;
        }
        SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
        return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
                            Cond);
      }
    }
  }
}

// FIXME: Support vectors.
if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
  const APInt &C1 = N1C->getAPIntValue();

  // (zext x) == C --> x == (trunc C)
  // (sext x) == C --> x == (trunc C)
  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
      DCI.isBeforeLegalize() && N0->hasOneUse()) {
    unsigned MinBits = N0.getValueSizeInBits();
    SDValue PreExt;
    bool Signed = false;
    if (N0->getOpcode() == ISD::ZERO_EXTEND) {
      // ZExt
      MinBits = N0->getOperand(0).getValueSizeInBits();
      PreExt = N0->getOperand(0);
    } else if (N0->getOpcode() == ISD::AND) {
      // DAGCombine turns costly ZExts into ANDs
      if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
        if ((C->getAPIntValue()+1).isPowerOf2()) {
          MinBits = C->getAPIntValue().countTrailingOnes();
          PreExt = N0->getOperand(0);
        }
    } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
      // SExt
      MinBits = N0->getOperand(0).getValueSizeInBits();
      PreExt = N0->getOperand(0);
      Signed = true;
    } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
      // ZEXTLOAD / SEXTLOAD
      if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
        MinBits = LN0->getMemoryVT().getSizeInBits();
        PreExt = N0;
      } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
        Signed = true;
        MinBits = LN0->getMemoryVT().getSizeInBits();
        PreExt = N0;
      }
    }

    // Figure out how many bits we need to preserve this constant.
    unsigned ReqdBits = Signed ?
      C1.getBitWidth() - C1.getNumSignBits() + 1 :
      C1.getActiveBits();

    // Make sure we're not losing bits from the constant.
    if (MinBits > 0 &&
        MinBits < C1.getBitWidth() &&
        MinBits >= ReqdBits) {
      EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
      if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
        // Will get folded away.
        SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
        if (MinBits == 1 && C1 == 1)
          // Invert the condition.
          return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
                              Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
        SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
        return DAG.getSetCC(dl, VT, Trunc, C, Cond);
      }

      // If truncating the setcc operands is not desirable, we can still
      // simplify the expression in some cases:
      // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
      // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
      // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
      // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
      // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
      // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
      SDValue TopSetCC = N0->getOperand(0);
      unsigned N0Opc = N0->getOpcode();
      bool SExt = (N0Opc == ISD::SIGN_EXTEND);
      if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
          TopSetCC.getOpcode() == ISD::SETCC &&
          (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
          (isConstFalseVal(N1C) ||
           isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

        bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
                       (!N1C->isZero() && Cond == ISD::SETNE);

        if (!Inverse)
          return TopSetCC;

        ISD::CondCode InvCond = ISD::getSetCCInverse(
            cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
            TopSetCC.getOperand(0).getValueType());
        return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
                                    TopSetCC.getOperand(1),
                                    InvCond);
      }
    }
  }

  // If the LHS is '(and load, const)', the RHS is 0, the test is for
  // equality or unsigned, and all 1 bits of the const are in the same
  // partial word, see if we can shorten the load.
  if (DCI.isBeforeLegalize() &&
      !ISD::isSignedIntSetCC(Cond) &&
      N0.getOpcode() == ISD::AND && C1 == 0 &&
      N0.getNode()->hasOneUse() &&
      isa<LoadSDNode>(N0.getOperand(0)) &&
      N0.getOperand(0).getNode()->hasOneUse() &&
      isa<ConstantSDNode>(N0.getOperand(1))) {
    LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
    APInt bestMask;
    unsigned bestWidth = 0, bestOffset = 0;
    if (Lod->isSimple() && Lod->isUnindexed()) {
      unsigned origWidth = N0.getValueSizeInBits();
      unsigned maskWidth = origWidth;
      // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
      // 8 bits, but have to be careful...
      if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
        origWidth = Lod->getMemoryVT().getSizeInBits();
      const APInt &Mask = N0.getConstantOperandAPInt(1);
      for (unsigned width = origWidth / 2; width>=8; width /= 2) {
        APInt newMask = APInt::getLowBitsSet(maskWidth, width);
        for (unsigned offset=0; offset<origWidth/width; offset++) {
          if (Mask.isSubsetOf(newMask)) {
            if (Layout.isLittleEndian())
              bestOffset = (uint64_t)offset * (width/8);
            else
              bestOffset = (origWidth/width - offset - 1) * (width/8);
            bestMask = Mask.lshr(offset * (width/8) * 8);
            bestWidth = width;
            break;
          }
          newMask <<= width;
        }
      }
    }
    if (bestWidth) {
      EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
      if (newVT.isRound() &&
          shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
        SDValue Ptr = Lod->getBasePtr();
        if (bestOffset != 0)
          Ptr =
              DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
        SDValue NewLoad =
            DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
                        Lod->getPointerInfo().getWithOffset(bestOffset),
                        Lod->getOriginalAlign());
        return DAG.getSetCC(dl, VT,
                            DAG.getNode(ISD::AND, dl, newVT, NewLoad,
                                    DAG.getConstant(bestMask.trunc(bestWidth),
                                                    dl, newVT)),
                            DAG.getConstant(0LL, dl, newVT), Cond);
      }
    }
  }

  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
    unsigned InSize = N0.getOperand(0).getValueSizeInBits();

    // If the comparison constant has bits in the upper part, the
    // zero-extended value could never match.
    if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
                                            C1.getBitWidth() - InSize))) {
      switch (Cond) {
      case ISD::SETUGT:
      case ISD::SETUGE:
      case ISD::SETEQ:
        return DAG.getConstant(0, dl, VT);
      case ISD::SETULT:
      case ISD::SETULE:
      case ISD::SETNE:
        return DAG.getConstant(1, dl, VT);
      case ISD::SETGT:
      case ISD::SETGE:
        // True if the sign bit of C1 is set.
        return DAG.getConstant(C1.isNegative(), dl, VT);
      case ISD::SETLT:
      case ISD::SETLE:
        // True if the sign bit of C1 isn't set.
        return DAG.getConstant(C1.isNonNegative(), dl, VT);
      default:
        break;
      }
    }

    // Otherwise, we can perform the comparison with the low bits.
    switch (Cond) {
    case ISD::SETEQ:
    case ISD::SETNE:
    case ISD::SETUGT:
    case ISD::SETUGE:
    case ISD::SETULT:
    case ISD::SETULE: {
      EVT newVT = N0.getOperand(0).getValueType();
      if (DCI.isBeforeLegalizeOps() ||
          (isOperationLegal(ISD::SETCC, newVT) &&
           isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
        EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
        SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

        SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
                                        NewConst, Cond);
        return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
      }
      break;
    }
    default:
      break; // todo, be more careful with signed comparisons
    }
  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
             (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
             !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
                                    OpVT)) {
    EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
    unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
    EVT ExtDstTy = N0.getValueType();
    unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

    // If the constant doesn't fit into the number of bits for the source of
    // the sign extension, it is impossible for both sides to be equal.
    if (C1.getMinSignedBits() > ExtSrcTyBits)
      return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);

    assert(ExtDstTy == N0.getOperand(0).getValueType() &&(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3793, __extension__ __PRETTY_FUNCTION__))
           ExtDstTy != ExtSrcTy && "Unexpected types!")(static_cast <bool> (ExtDstTy == N0.getOperand(0).getValueType
() && ExtDstTy != ExtSrcTy && "Unexpected types!"
) ? void (0) : __assert_fail ("ExtDstTy == N0.getOperand(0).getValueType() && ExtDstTy != ExtSrcTy && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3793, __extension__ __PRETTY_FUNCTION__));
    APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
    SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
                                 DAG.getConstant(Imm, dl, ExtDstTy));
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(ZextOp.getNode());
    // Otherwise, make this a use of a zext.
    return DAG.getSetCC(dl, VT, ZextOp,
                        DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
  } else if ((N1C->isZero() || N1C->isOne()) &&
             (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
    // SETCC (SETCC), [0|1], [EQ|NE]  -> SETCC
    if (N0.getOpcode() == ISD::SETCC &&
        isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
        (N0.getValueType() == MVT::i1 ||
         getBooleanContents(N0.getOperand(0).getValueType()) ==
                     ZeroOrOneBooleanContent)) {
      bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
      if (TrueWhenTrue)
        return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
      // Invert the condition.
      ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
      CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
      if (DCI.isBeforeLegalizeOps() ||
          isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
        return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
    }

    if ((N0.getOpcode() == ISD::XOR ||
         (N0.getOpcode() == ISD::AND &&
          N0.getOperand(0).getOpcode() == ISD::XOR &&
          N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
        isOneConstant(N0.getOperand(1))) {
      // If this is (X^1) == 0/1, swap the RHS and eliminate the xor.  We
      // can only do this if the top bits are known zero.
      unsigned BitWidth = N0.getValueSizeInBits();
      if (DAG.MaskedValueIsZero(N0,
                                APInt::getHighBitsSet(BitWidth,
                                                      BitWidth-1))) {
        // Okay, get the un-inverted input value.
        SDValue Val;
        if (N0.getOpcode() == ISD::XOR) {
          Val = N0.getOperand(0);
        } else {
          assert(N0.getOpcode() == ISD::AND &&(static_cast <bool> (N0.getOpcode() == ISD::AND &&
 N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
 ("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3838, __extension__ __PRETTY_FUNCTION__))
                  N0.getOperand(0).getOpcode() == ISD::XOR)(static_cast <bool> (N0.getOpcode() == ISD::AND &&
 N0.getOperand(0).getOpcode() == ISD::XOR) ? void (0) : __assert_fail
 ("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3838, __extension__ __PRETTY_FUNCTION__));
          // ((X^1)&1)^1 -> X & 1
          Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
                            N0.getOperand(0).getOperand(0),
                            N0.getOperand(1));
        }

        return DAG.getSetCC(dl, VT, Val, N1,
                            Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
      }
    } else if (N1C->isOne()) {
      SDValue Op0 = N0;
      if (Op0.getOpcode() == ISD::TRUNCATE)
        Op0 = Op0.getOperand(0);

      if ((Op0.getOpcode() == ISD::XOR) &&
          Op0.getOperand(0).getOpcode() == ISD::SETCC &&
          Op0.getOperand(1).getOpcode() == ISD::SETCC) {
        SDValue XorLHS = Op0.getOperand(0);
        SDValue XorRHS = Op0.getOperand(1);
        // Ensure that the input setccs return an i1 type or 0/1 value.
        if (Op0.getValueType() == MVT::i1 ||
            (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
                    ZeroOrOneBooleanContent &&
             getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
                      ZeroOrOneBooleanContent)) {
          // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
          Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
          return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
        }
      }
      if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
        // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
        if (Op0.getValueType().bitsGT(VT))
          Op0 = DAG.getNode(ISD::AND, dl, VT,
                        DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
                        DAG.getConstant(1, dl, VT));
        else if (Op0.getValueType().bitsLT(VT))
          Op0 = DAG.getNode(ISD::AND, dl, VT,
                      DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
                      DAG.getConstant(1, dl, VT));

        return DAG.getSetCC(dl, VT, Op0,
                            DAG.getConstant(0, dl, Op0.getValueType()),
                            Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
      }
      if (Op0.getOpcode() == ISD::AssertZext &&
          cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
        return DAG.getSetCC(dl, VT, Op0,
                            DAG.getConstant(0, dl, Op0.getValueType()),
                            Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
    }
  }

  // Given:
  //   icmp eq/ne (urem %x, %y), 0
  // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
  //   icmp eq/ne %x, 0
  if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
      (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
    KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
    KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
    if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
      return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
  }

  // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
  //  and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
      N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
      N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
      N1C && N1C->isAllOnes()) {
    return DAG.getSetCC(dl, VT, N0.getOperand(0),
                        DAG.getConstant(0, dl, OpVT),
                        Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
  }

  if (SDValue V =
          optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
    return V;
}

// These simplifications apply to splat vectors as well.
// TODO: Handle more splat vector cases.
if (auto *N1C = isConstOrConstSplat(N1)) {
  const APInt &C1 = N1C->getAPIntValue();

  APInt MinVal, MaxVal;
  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
  if (ISD::isSignedIntSetCC(Cond)) {
    MinVal = APInt::getSignedMinValue(OperandBitSize);
    MaxVal = APInt::getSignedMaxValue(OperandBitSize);
  } else {
    MinVal = APInt::getMinValue(OperandBitSize);
    MaxVal = APInt::getMaxValue(OperandBitSize);
  }

  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
    // X >= MIN --> true
    if (C1 == MinVal)
      return DAG.getBoolConstant(true, dl, VT, OpVT);

    if (!VT.isVector()) { // TODO: Support this for vectors.
      // X >= C0 --> X > (C0 - 1)
      APInt C = C1 - 1;
      ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
      if ((DCI.isBeforeLegalizeOps() ||
           isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
          (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
                                isLegalICmpImmediate(C.getSExtValue())))) {
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(C, dl, N1.getValueType()),
                            NewCC);
      }
    }
  }

  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
    // X <= MAX --> true
    if (C1 == MaxVal)
      return DAG.getBoolConstant(true, dl, VT, OpVT);

    // X <= C0 --> X < (C0 + 1)
    if (!VT.isVector()) { // TODO: Support this for vectors.
      APInt C = C1 + 1;
      ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
      if ((DCI.isBeforeLegalizeOps() ||
           isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
          (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
                                isLegalICmpImmediate(C.getSExtValue())))) {
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(C, dl, N1.getValueType()),
                            NewCC);
      }
    }
  }

  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
    if (C1 == MinVal)
      return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

    // TODO: Support this for vectors after legalize ops.
    if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
      // Canonicalize setlt X, Max --> setne X, Max
      if (C1 == MaxVal)
        return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

      // If we have setult X, 1, turn it into seteq X, 0
      if (C1 == MinVal+1)
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(MinVal, dl, N0.getValueType()),
                            ISD::SETEQ);
    }
  }

  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
    if (C1 == MaxVal)
      return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

    // TODO: Support this for vectors after legalize ops.
    if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
      // Canonicalize setgt X, Min --> setne X, Min
      if (C1 == MinVal)
        return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

      // If we have setugt X, Max-1, turn it into seteq X, Max
      if (C1 == MaxVal-1)
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(MaxVal, dl, N0.getValueType()),
                            ISD::SETEQ);
    }
  }

  if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
    // (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
    if (C1.isZero())
      if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
              VT, N0, N1, Cond, DCI, dl))
        return CC;

    // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
    // For example, when high 32-bits of i64 X are known clear:
    // all bits clear: (X | (Y<<32)) ==  0 --> (X | Y) ==  0
    // all bits set:   (X | (Y<<32)) == -1 --> (X & Y) == -1
    bool CmpZero = N1C->getAPIntValue().isZero();
    bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
    if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
      // Match or(lo,shl(hi,bw/2)) pattern.
      auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
        unsigned EltBits = V.getScalarValueSizeInBits();
        if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
          return false;
        SDValue LHS = V.getOperand(0);
        SDValue RHS = V.getOperand(1);
        APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
        // Unshifted element must have zero upperbits.
        if (RHS.getOpcode() == ISD::SHL &&
            isa<ConstantSDNode>(RHS.getOperand(1)) &&
            RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
            DAG.MaskedValueIsZero(LHS, HiBits)) {
          Lo = LHS;
          Hi = RHS.getOperand(0);
          return true;
        }
        if (LHS.getOpcode() == ISD::SHL &&
            isa<ConstantSDNode>(LHS.getOperand(1)) &&
            LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
            DAG.MaskedValueIsZero(RHS, HiBits)) {
          Lo = RHS;
          Hi = LHS.getOperand(0);
          return true;
        }
        return false;
      };

      auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
        unsigned EltBits = N0.getScalarValueSizeInBits();
        unsigned HalfBits = EltBits / 2;
        APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
        SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
        SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
        SDValue NewN0 =
            DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
        SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
        return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
      };

      SDValue Lo, Hi;
      if (IsConcat(N0, Lo, Hi))
        return MergeConcat(Lo, Hi);

      if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
        SDValue Lo0, Lo1, Hi0, Hi1;
        if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
            IsConcat(N0.getOperand(1), Lo1, Hi1)) {
          return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
                             DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
        }
      }
    }
  }

  // If we have "setcc X, C0", check to see if we can shrink the immediate
  // by changing cc.
  // TODO: Support this for vectors after legalize ops.
  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
    // SETUGT X, SINTMAX  -> SETLT X, 0
    // SETUGE X, SINTMIN -> SETLT X, 0
    if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
        (Cond == ISD::SETUGE && C1.isMinSignedValue()))
      return DAG.getSetCC(dl, VT, N0,
                          DAG.getConstant(0, dl, N1.getValueType()),
                          ISD::SETLT);

    // SETULT X, SINTMIN  -> SETGT X, -1
    // SETULE X, SINTMAX  -> SETGT X, -1
    if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
        (Cond == ISD::SETULE && C1.isMaxSignedValue()))
      return DAG.getSetCC(dl, VT, N0,
                          DAG.getAllOnesConstant(dl, N1.getValueType()),
                          ISD::SETGT);
  }
}

// Back to non-vector simplifications.
// TODO: Can we do these for vector splats?
if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  const APInt &C1 = N1C->getAPIntValue();
  EVT ShValTy = N0.getValueType();

  // Fold bit comparisons when we can. This will result in an
  // incorrect value when boolean false is negative one, unless
  // the bitsize is 1 in which case the false value is the same
  // in practice regardless of the representation.
  if ((VT.getSizeInBits() == 1 ||
       getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
      (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
      (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
      N0.getOpcode() == ISD::AND) {
    if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
      EVT ShiftTy =
          getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
      if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0  -->  (X & 8) >> 3
        // Perform the xform if the AND RHS is a single bit.
        unsigned ShCt = AndRHS->getAPIntValue().logBase2();
        if (AndRHS->getAPIntValue().isPowerOf2() &&
            !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
          return DAG.getNode(ISD::TRUNCATE, dl, VT,
                             DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                                         DAG.getConstant(ShCt, dl, ShiftTy)));
        }
      } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
        // (X & 8) == 8  -->  (X & 8) >> 3
        // Perform the xform if C1 is a single bit.
        unsigned ShCt = C1.logBase2();
        if (C1.isPowerOf2() &&
            !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
          return DAG.getNode(ISD::TRUNCATE, dl, VT,
                             DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                                         DAG.getConstant(ShCt, dl, ShiftTy)));
        }
      }
    }
  }

  if (C1.getMinSignedBits() <= 64 &&
      !isLegalICmpImmediate(C1.getSExtValue())) {
    EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
    // (X & -256) == 256 -> (X >> 8) == 1
    if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
      if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
        const APInt &AndRHSC = AndRHS->getAPIntValue();
        if (AndRHSC.isNegatedPowerOf2() && (AndRHSC & C1) == C1) {
          unsigned ShiftBits = AndRHSC.countTrailingZeros();
          if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
            SDValue Shift =
              DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
                          DAG.getConstant(ShiftBits, dl, ShiftTy));
            SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
            return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
          }
        }
      }
    } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
               Cond == ISD::SETULE || Cond == ISD::SETUGT) {
      bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
      // X <  0x100000000 -> (X >> 32) <  1
      // X >= 0x100000000 -> (X >> 32) >= 1
      // X <= 0x0ffffffff -> (X >> 32) <  1
      // X >  0x0ffffffff -> (X >> 32) >= 1
      unsigned ShiftBits;
      APInt NewC = C1;
      ISD::CondCode NewCond = Cond;
      if (AdjOne) {
        ShiftBits = C1.countTrailingOnes();
        NewC = NewC + 1;
        NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
      } else {
        ShiftBits = C1.countTrailingZeros();
      }
      NewC.lshrInPlace(ShiftBits);
      if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
          isLegalICmpImmediate(NewC.getSExtValue()) &&
          !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
        SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                                    DAG.getConstant(ShiftBits, dl, ShiftTy));
        SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
        return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
      }
    }
  }
}

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
  auto *CFP = cast<ConstantFPSDNode>(N1);
  assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")(static_cast <bool> (!CFP->getValueAPF().isNaN() &&
 "Unexpected NaN value") ? void (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4196, __extension__ __PRETTY_FUNCTION__));

  // Otherwise, we know the RHS is not a NaN.  Simplify the node to drop the
  // constant if knowing that the operand is non-nan is enough.  We prefer to
  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
  // materialize 0.0.
  if (Cond == ISD::SETO || Cond == ISD::SETUO)
    return DAG.getSetCC(dl, VT, N0, N0, Cond);

  // setcc (fneg x), C -> setcc swap(pred) x, -C
  if (N0.getOpcode() == ISD::FNEG) {
    ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
    if (DCI.isBeforeLegalizeOps() ||
        isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
      SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
      return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
    }
  }

  // If the condition is not legal, see if we can find an equivalent one
  // which is legal.
  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
    // If the comparison was an awkward floating-point == or != and one of
    // the comparison operands is infinity or negative infinity, convert the
    // condition to a less-awkward <= or >=.
    if (CFP->getValueAPF().isInfinity()) {
      bool IsNegInf = CFP->getValueAPF().isNegative();
      ISD::CondCode NewCond = ISD::SETCC_INVALID;
      switch (Cond) {
      case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
      case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
      case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
      case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
      default: break;
      }
      if (NewCond != ISD::SETCC_INVALID &&
          isCondCodeLegal(NewCond, N0.getSimpleValueType()))
        return DAG.getSetCC(dl, VT, N0, N1, NewCond);
    }
  }
}

if (N0 == N1) {
  // The sext(setcc()) => setcc() optimization relies on the appropriate
  // constant being emitted.
  assert(!N0.getValueType().isInteger() &&(static_cast <bool> (!N0.getValueType().isInteger() &&
 "Integer types should be handled by FoldSetCC") ? void (0) :
 __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4242, __extension__ __PRETTY_FUNCTION__))
         "Integer types should be handled by FoldSetCC")(static_cast <bool> (!N0.getValueType().isInteger() &&
 "Integer types should be handled by FoldSetCC") ? void (0) :
 __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4242, __extension__ __PRETTY_FUNCTION__));

  bool EqTrue = ISD::isTrueWhenEqual(Cond);
  unsigned UOF = ISD::getUnorderedFlavor(Cond);
  if (UOF == 2) // FP operators that are undefined on NaNs.
    return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
  if (UOF == unsigned(EqTrue))
    return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
  // Otherwise, we can't fold it.  However, we can simplify it to SETUO/SETO
  // if it is not already.
  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
  if (NewCond != Cond &&
      (DCI.isBeforeLegalizeOps() ||
                          isCondCodeLegal(NewCond, N0.getSimpleValueType())))
    return DAG.getSetCC(dl, VT, N0, N1, NewCond);
}

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
    N0.getValueType().isInteger()) {
  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
      N0.getOpcode() == ISD::XOR) {
    // Simplify (X+Y) == (X+Z) -->  Y == Z
    if (N0.getOpcode() == N1.getOpcode()) {
      if (N0.getOperand(0) == N1.getOperand(0))
        return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
      if (N0.getOperand(1) == N1.getOperand(1))
        return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
      if (isCommutativeBinOp(N0.getOpcode())) {
        // If X op Y == Y op X, try other combinations.
        if (N0.getOperand(0) == N1.getOperand(1))
          return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
                              Cond);
        if (N0.getOperand(1) == N1.getOperand(0))
          return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
                              Cond);
      }
    }

    // If RHS is a legal immediate value for a compare instruction, we need
    // to be careful about increasing register pressure needlessly.
    bool LegalRHSImm = false;

    if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
      if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
        // Turn (X+C1) == C2 --> X == C2-C1
        if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
          return DAG.getSetCC(dl, VT, N0.getOperand(0),
                              DAG.getConstant(RHSC->getAPIntValue()-
                                              LHSR->getAPIntValue(),
                              dl, N0.getValueType()), Cond);
        }

        // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
        if (N0.getOpcode() == ISD::XOR)
          // If we know that all of the inverted bits are zero, don't bother
          // performing the inversion.
          if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
            return
              DAG.getSetCC(dl, VT, N0.getOperand(0),
                           DAG.getConstant(LHSR->getAPIntValue() ^
                                             RHSC->getAPIntValue(),
                                           dl, N0.getValueType()),
                           Cond);
      }

      // Turn (C1-X) == C2 --> X == C1-C2
      if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
        if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
          return
            DAG.getSetCC(dl, VT, N0.getOperand(1),
                         DAG.getConstant(SUBC->getAPIntValue() -
                                           RHSC->getAPIntValue(),
                                         dl, N0.getValueType()),
                         Cond);
        }
      }

      // Could RHSC fold directly into a compare?
      if (RHSC->getValueType(0).getSizeInBits() <= 64)
        LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
    }

    // (X+Y) == X --> Y == 0 and similar folds.
    // Don't do this if X is an immediate that can fold into a cmp
    // instruction and X+Y has other uses. It could be an induction variable
    // chain, and the transform would increase register pressure.
    if (!LegalRHSImm || N0.hasOneUse())
      if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
        return V;
  }

  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
      N1.getOpcode() == ISD::XOR)
    if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
      return V;

  if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
    return V;
}

// Fold remainder of division by a constant.
if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
    N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

  // When division is cheap or optimizing for minimum size,
  // fall through to DIVREM creation by skipping this fold.
  if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
    if (N0.getOpcode() == ISD::UREM) {
      if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
        return Folded;
    } else if (N0.getOpcode() == ISD::SREM) {
      if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
        return Folded;
    }
  }
}

// Fold away ALL boolean setcc's.
if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
  SDValue Temp;
  switch (Cond) {
  default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4364);
  case ISD::SETEQ:  // X == Y  -> ~(X^Y)
    Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
    N0 = DAG.getNOT(dl, Temp, OpVT);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETNE:  // X != Y   -->  (X^Y)
    N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
    break;
  case ISD::SETGT:  // X >s Y   -->  X == 0 & Y == 1  -->  ~X & Y
  case ISD::SETULT: // X <u Y   -->  X == 0 & Y == 1  -->  ~X & Y
    Temp = DAG.getNOT(dl, N0, OpVT);
    N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETLT:  // X <s Y   --> X == 1 & Y == 0  -->  ~Y & X
  case ISD::SETUGT: // X >u Y   --> X == 1 & Y == 0  -->  ~Y & X
    Temp = DAG.getNOT(dl, N1, OpVT);
    N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETULE: // X <=u Y  --> X == 0 | Y == 1  -->  ~X | Y
  case ISD::SETGE:  // X >=s Y  --> X == 0 | Y == 1  -->  ~X | Y
    Temp = DAG.getNOT(dl, N0, OpVT);
    N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETUGE: // X >=u Y  --> X == 1 | Y == 0  -->  ~Y | X
  case ISD::SETLE:  // X <=s Y  --> X == 1 | Y == 0  -->  ~Y | X
    Temp = DAG.getNOT(dl, N1, OpVT);
    N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
    break;
  }
  if (VT.getScalarType() != MVT::i1) {
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(N0.getNode());
    // FIXME: If running after legalize, we probably can't do this.
    ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
    N0 = DAG.getNode(ExtendCode, dl, VT, N0);
  }
  return N0;
}

// Could not fold it.
return SDValue();
4413}

4415/// Returns true (and the GlobalValue and the offset) if the node is a
4416/// GlobalAddress + offset.
4417bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
                                  int64_t &Offset) const {

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
  GA = GASD->getGlobal();
  Offset += GASD->getOffset();
  return true;
}

if (N->getOpcode() == ISD::ADD) {
  SDValue N1 = N->getOperand(0);
  SDValue N2 = N->getOperand(1);
  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
    if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
      Offset += V->getSExtValue();
      return true;
    }
  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
    if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
      Offset += V->getSExtValue();
      return true;
    }
  }
}

return false;
4445}

4447SDValue TargetLowering::PerformDAGCombine(SDNode *N,
                                        DAGCombinerInfo &DCI) const {
// Default implementation: no optimization.
return SDValue();
4451}

4453//===----------------------------------------------------------------------===//
4454//  Inline Assembler Implementation Methods
4455//===----------------------------------------------------------------------===//

4457TargetLowering::ConstraintType
4458TargetLowering::getConstraintType(StringRef Constraint) const {
unsigned S = Constraint.size();

if (S == 1) {
  switch (Constraint[0]) {
  default: break;
  case 'r':
    return C_RegisterClass;
  case 'm': // memory
  case 'o': // offsetable
  case 'V': // not offsetable
    return C_Memory;
  case 'n': // Simple Integer
  case 'E': // Floating Point Constant
  case 'F': // Floating Point Constant
    return C_Immediate;
  case 'i': // Simple Integer or Relocatable Constant
  case 's': // Relocatable Constant
  case 'p': // Address.
  case 'X': // Allow ANY value.
  case 'I': // Target registers.
  case 'J':
  case 'K':
  case 'L':
  case 'M':
  case 'N':
  case 'O':
  case 'P':
  case '<':
  case '>':
    return C_Other;
  }
}

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
    return C_Memory;
  return C_Register;
}
return C_Unknown;
4498}

4500/// Try to replace an X constraint, which matches anything, with another that
4501/// has more specific requirements based on the type of the corresponding
4502/// operand.
4503const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
if (ConstraintVT.isInteger())
  return "r";
if (ConstraintVT.isFloatingPoint())
  return "f"; // works for many targets
return nullptr;
4509}

4511SDValue TargetLowering::LowerAsmOutputForConstraint(
  SDValue &Chain, SDValue &Flag, const SDLoc &DL,
  const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
return SDValue();
4515}

4517/// Lower the specified operand into the Ops vector.
4518/// If it is invalid, don't add anything to Ops.
4519void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
                                                std::string &Constraint,
                                                std::vector<SDValue> &Ops,
                                                SelectionDAG &DAG) const {

if (Constraint.length() > 1) return;

char ConstraintLetter = Constraint[0];
switch (ConstraintLetter) {
default: break;
case 'X':     // Allows any operand; labels (basic block) use this.
  if (Op.getOpcode() == ISD::BasicBlock ||
      Op.getOpcode() == ISD::TargetBlockAddress) {
    Ops.push_back(Op);
    return;
  }
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case 'i':    // Simple Integer or Relocatable Constant
case 'n':    // Simple Integer
case 's': {  // Relocatable Constant

  GlobalAddressSDNode *GA;
  ConstantSDNode *C;
  BlockAddressSDNode *BA;
  uint64_t Offset = 0;

  // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
  // etc., since getelementpointer is variadic. We can't use
  // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
  // while in this case the GA may be furthest from the root node which is
  // likely an ISD::ADD.
  while (1) {
    if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
      Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
                                               GA->getValueType(0),
                                               Offset + GA->getOffset()));
      return;
    }
    if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
      // gcc prints these as sign extended.  Sign extend value to 64 bits
      // now; without this it would get ZExt'd later in
      // ScheduleDAGSDNodes::EmitNode, which is very generic.
      bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
      BooleanContent BCont = getBooleanContents(MVT::i64);
      ISD::NodeType ExtOpc =
          IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
      int64_t ExtVal =
          ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
      Ops.push_back(
          DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
      return;
    }
    if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {
      Ops.push_back(DAG.getTargetBlockAddress(
          BA->getBlockAddress(), BA->getValueType(0),
          Offset + BA->getOffset(), BA->getTargetFlags()));
      return;
    }
    const unsigned OpCode = Op.getOpcode();
    if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
      if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
        Op = Op.getOperand(1);
      // Subtraction is not commutative.
      else if (OpCode == ISD::ADD &&
               (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
        Op = Op.getOperand(0);
      else
        return;
      Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
      continue;
    }
    return;
  }
  break;
}
}
4595}

4597std::pair<unsigned, const TargetRegisterClass *>
4598TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
                                           StringRef Constraint,
                                           MVT VT) const {
if (Constraint.empty() || Constraint[0] != '{')
  return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")(static_cast <bool> (*(Constraint.end() - 1) == '}' &&
 "Not a brace enclosed constraint?") ? void (0) : __assert_fail
 ("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4603, __extension__ __PRETTY_FUNCTION__));

// Remove the braces from around the name.
StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

std::pair<unsigned, const TargetRegisterClass *> R =
    std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

// Figure out which register class contains this reg.
for (const TargetRegisterClass *RC : RI->regclasses()) {
  // If none of the value types for this register class are valid, we
  // can't use it.  For example, 64-bit reg classes on 32-bit targets.
  if (!isLegalRC(*RI, *RC))
    continue;

  for (const MCPhysReg &PR : *RC) {
    if (RegName.equals_insensitive(RI->getRegAsmName(PR))) {
      std::pair<unsigned, const TargetRegisterClass *> S =
          std::make_pair(PR, RC);

      // If this register class has the requested value type, return it,
      // otherwise keep searching and return the first class found
      // if no other is found which explicitly has the requested type.
      if (RI->isTypeLegalForClass(*RC, VT))
        return S;
      if (!R.second)
        R = S;
    }
  }
}

return R;
4635}

4637//===----------------------------------------------------------------------===//
4638// Constraint Selection.

4640/// Return true of this is an input operand that is a matching constraint like
4641/// "4".
4642bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
 "No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4643, __extension__ __PRETTY_FUNCTION__));
return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4645}

4647/// If this is an input matching constraint, this method returns the output
4648/// operand it matches.
4649unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
assert(!ConstraintCode.empty() && "No known constraint!")(static_cast <bool> (!ConstraintCode.empty() &&
 "No known constraint!") ? void (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4650, __extension__ __PRETTY_FUNCTION__));
return atoi(ConstraintCode.c_str());
4652}

4654/// Split up the constraint string from the inline assembly value into the
4655/// specific constraints and their prefixes, and also tie in the associated
4656/// operand values.
4657/// If this returns an empty vector, and if the constraint string itself
4658/// isn't empty, there was an error parsing.
4659TargetLowering::AsmOperandInfoVector
4660TargetLowering::ParseConstraints(const DataLayout &DL,
                               const TargetRegisterInfo *TRI,
                               const CallBase &Call) const {
/// Information about all of the constraints.
AsmOperandInfoVector ConstraintOperands;
const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
unsigned maCount = 0; // Largest number of multiple alternative constraints.

// Do a prepass over the constraints, canonicalizing them, and building up the
// ConstraintOperands list.
unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
unsigned ResNo = 0; // ResNo - The result number of the next output.

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
  ConstraintOperands.emplace_back(std::move(CI));
  AsmOperandInfo &OpInfo = ConstraintOperands.back();

  // Update multiple alternative constraint count.
  if (OpInfo.multipleAlternatives.size() > maCount)
    maCount = OpInfo.multipleAlternatives.size();

  OpInfo.ConstraintVT = MVT::Other;

  // Compute the value type for each operand.
  switch (OpInfo.Type) {
  case InlineAsm::isOutput:
    // Indirect outputs just consume an argument.
    if (OpInfo.isIndirect) {
      OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
      break;
    }

    // The return value of the call is this value.  As such, there is no
    // corresponding argument.
    assert(!Call.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!Call.getType()->isVoidTy() &&
 "Bad inline asm!") ? void (0) : __assert_fail ("!Call.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4694, __extension__ __PRETTY_FUNCTION__));
    if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
      OpInfo.ConstraintVT =
          getSimpleValueType(DL, STy->getElementType(ResNo));
    } else {
      assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4699, __extension__ __PRETTY_FUNCTION__));
      OpInfo.ConstraintVT =
          getAsmOperandValueType(DL, Call.getType()).getSimpleVT();
    }
    ++ResNo;
    break;
  case InlineAsm::isInput:
    OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
    break;
  case InlineAsm::isClobber:
    // Nothing to do.
    break;
  }

  if (OpInfo.CallOperandVal) {
    llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
    if (OpInfo.isIndirect) {
      llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
      if (!PtrTy)
        report_fatal_error("Indirect operand for inline asm not a pointer!");
      OpTy = PtrTy->getElementType();
    }

    // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
    if (StructType *STy = dyn_cast<StructType>(OpTy))
      if (STy->getNumElements() == 1)
        OpTy = STy->getElementType(0);

    // If OpTy is not a single value, it may be a struct/union that we
    // can tile with integers.
    if (!OpTy->isSingleValueType() && OpTy->isSized()) {
      unsigned BitSize = DL.getTypeSizeInBits(OpTy);
      switch (BitSize) {
      default: break;
      case 1:
      case 8:
      case 16:
      case 32:
      case 64:
      case 128:
        OpInfo.ConstraintVT =
            MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
        break;
      }
    } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
      unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
      OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
    } else {
      OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
    }
  }
}

// If we have multiple alternative constraints, select the best alternative.
if (!ConstraintOperands.empty()) {
  if (maCount) {
    unsigned bestMAIndex = 0;
    int bestWeight = -1;
    // weight:  -1 = invalid match, and 0 = so-so match to 5 = good match.
    int weight = -1;
    unsigned maIndex;
    // Compute the sums of the weights for each alternative, keeping track
    // of the best (highest weight) one so far.
    for (maIndex = 0; maIndex < maCount; ++maIndex) {
      int weightSum = 0;
      for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
           cIndex != eIndex; ++cIndex) {
        AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
        if (OpInfo.Type == InlineAsm::isClobber)
          continue;

        // If this is an output operand with a matching input operand,
        // look up the matching input. If their types mismatch, e.g. one
        // is an integer, the other is floating point, or their sizes are
        // different, flag it as an maCantMatch.
        if (OpInfo.hasMatchingInput()) {
          AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
          if (OpInfo.ConstraintVT != Input.ConstraintVT) {
            if ((OpInfo.ConstraintVT.isInteger() !=
                 Input.ConstraintVT.isInteger()) ||
                (OpInfo.ConstraintVT.getSizeInBits() !=
                 Input.ConstraintVT.getSizeInBits())) {
              weightSum = -1; // Can't match.
              break;
            }
          }
        }
        weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
        if (weight == -1) {
          weightSum = -1;
          break;
        }
        weightSum += weight;
      }
      // Update best.
      if (weightSum > bestWeight) {
        bestWeight = weightSum;
        bestMAIndex = maIndex;
      }
    }

    // Now select chosen alternative in each constraint.
    for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
         cIndex != eIndex; ++cIndex) {
      AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
      if (cInfo.Type == InlineAsm::isClobber)
        continue;
      cInfo.selectAlternative(bestMAIndex);
    }
  }
}

// Check and hook up tied operands, choose constraint code to use.
for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
     cIndex != eIndex; ++cIndex) {
  AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

  // If this is an output operand with a matching input operand, look up the
  // matching input. If their types mismatch, e.g. one is an integer, the
  // other is floating point, or their sizes are different, flag it as an
  // error.
  if (OpInfo.hasMatchingInput()) {
    AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

    if (OpInfo.ConstraintVT != Input.ConstraintVT) {
      std::pair<unsigned, const TargetRegisterClass *> MatchRC =
          getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
                                       OpInfo.ConstraintVT);
      std::pair<unsigned, const TargetRegisterClass *> InputRC =
          getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
                                       Input.ConstraintVT);
      if ((OpInfo.ConstraintVT.isInteger() !=
           Input.ConstraintVT.isInteger()) ||
          (MatchRC.second != InputRC.second)) {
        report_fatal_error("Unsupported asm: input constraint"
                           " with a matching output constraint of"
                           " incompatible type!");
      }
    }
  }
}

return ConstraintOperands;
4842}

4844/// Return an integer indicating how general CT is.
4845static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
switch (CT) {
case TargetLowering::C_Immediate:
case TargetLowering::C_Other:
case TargetLowering::C_Unknown:
  return 0;
case TargetLowering::C_Register:
  return 1;
case TargetLowering::C_RegisterClass:
  return 2;
case TargetLowering::C_Memory:
  return 3;
}
llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4858);
4859}

4861/// Examine constraint type and operand type and determine a weight value.
4862/// This object must already have been set up with the operand type
4863/// and the current alternative constraint selected.
4864TargetLowering::ConstraintWeight
TargetLowering::getMultipleConstraintMatchWeight(
  AsmOperandInfo &info, int maIndex) const {
InlineAsm::ConstraintCodeVector *rCodes;
if (maIndex >= (int)info.multipleAlternatives.size())
  rCodes = &info.Codes;
else
  rCodes = &info.multipleAlternatives[maIndex].Codes;
ConstraintWeight BestWeight = CW_Invalid;

// Loop over the options, keeping track of the most general one.
for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
  ConstraintWeight weight =
    getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
  if (weight > BestWeight)
    BestWeight = weight;
}

return BestWeight;
4883}

4885/// Examine constraint type and operand type and determine a weight value.
4886/// This object must already have been set up with the operand type
4887/// and the current alternative constraint selected.
4888TargetLowering::ConstraintWeight
TargetLowering::getSingleConstraintMatchWeight(
  AsmOperandInfo &info, const char *constraint) const {
ConstraintWeight weight = CW_Invalid;
Value *CallOperandVal = info.CallOperandVal;
  // If we don't have a value, we can't do a match,
  // but allow it at the lowest weight.
if (!CallOperandVal)
  return CW_Default;
// Look at the constraint type.
switch (*constraint) {
  case 'i': // immediate integer.
  case 'n': // immediate integer with a known value.
    if (isa<ConstantInt>(CallOperandVal))
      weight = CW_Constant;
    break;
  case 's': // non-explicit intregal immediate.
    if (isa<GlobalValue>(CallOperandVal))
      weight = CW_Constant;
    break;
  case 'E': // immediate float if host format.
  case 'F': // immediate float.
    if (isa<ConstantFP>(CallOperandVal))
      weight = CW_Constant;
    break;
  case '<': // memory operand with autodecrement.
  case '>': // memory operand with autoincrement.
  case 'm': // memory operand.
  case 'o': // offsettable memory operand
  case 'V': // non-offsettable memory operand
    weight = CW_Memory;
    break;
  case 'r': // general register.
  case 'g': // general register, memory operand or immediate integer.
            // note: Clang converts "g" to "imr".
    if (CallOperandVal->getType()->isIntegerTy())
      weight = CW_Register;
    break;
  case 'X': // any operand.
default:
  weight = CW_Default;
  break;
}
return weight;
4932}

4934/// If there are multiple different constraints that we could pick for this
4935/// operand (e.g. "imr") try to pick the 'best' one.
4936/// This is somewhat tricky: constraints fall into four classes:
4937///    Other         -> immediates and magic values
4938///    Register      -> one specific register
4939///    RegisterClass -> a group of regs
4940///    Memory        -> memory
4941/// Ideally, we would pick the most specific constraint possible: if we have
4942/// something that fits into a register, we would pick it.  The problem here
4943/// is that if we have something that could either be in a register or in
4944/// memory that use of the register could cause selection of *other*
4945/// operands to fail: they might only succeed if we pick memory.  Because of
4946/// this the heuristic we use is:
4947///
4948///  1) If there is an 'other' constraint, and if the operand is valid for
4949///     that constraint, use it.  This makes us take advantage of 'i'
4950///     constraints when available.
4951///  2) Otherwise, pick the most general constraint present.  This prefers
4952///     'm' over 'r', for example.
4953///
4954static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
                           const TargetLowering &TLI,
                           SDValue Op, SelectionDAG *DAG) {
assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")(static_cast <bool> (OpInfo.Codes.size() > 1 &&
 "Doesn't have multiple constraint options") ? void (0) : __assert_fail
 ("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4957, __extension__ __PRETTY_FUNCTION__));
unsigned BestIdx = 0;
TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
int BestGenerality = -1;

// Loop over the options, keeping track of the most general one.
for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
  TargetLowering::ConstraintType CType =
    TLI.getConstraintType(OpInfo.Codes[i]);

  // Indirect 'other' or 'immediate' constraints are not allowed.
  if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
                             CType == TargetLowering::C_Register ||
                             CType == TargetLowering::C_RegisterClass))
    continue;

  // If this is an 'other' or 'immediate' constraint, see if the operand is
  // valid for it. For example, on X86 we might have an 'rI' constraint. If
  // the operand is an integer in the range [0..31] we want to use I (saving a
  // load of a register), otherwise we must use 'r'.
  if ((CType == TargetLowering::C_Other ||
       CType == TargetLowering::C_Immediate) && Op.getNode()) {
    assert(OpInfo.Codes[i].size() == 1 &&(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
 "Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
 ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__))
           "Unhandled multi-letter 'other' constraint")(static_cast <bool> (OpInfo.Codes[i].size() == 1 &&
 "Unhandled multi-letter 'other' constraint") ? void (0) : __assert_fail
 ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4980, __extension__ __PRETTY_FUNCTION__));
    std::vector<SDValue> ResultOps;
    TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
                                     ResultOps, *DAG);
    if (!ResultOps.empty()) {
      BestType = CType;
      BestIdx = i;
      break;
    }
  }

  // Things with matching constraints can only be registers, per gcc
  // documentation.  This mainly affects "g" constraints.
  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
    continue;

  // This constraint letter is more general than the previous one, use it.
  int Generality = getConstraintGenerality(CType);
  if (Generality > BestGenerality) {
    BestType = CType;
    BestIdx = i;
    BestGenerality = Generality;
  }
}

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
OpInfo.ConstraintType = BestType;
5007}

5009/// Determines the constraint code and constraint type to use for the specific
5010/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
5011void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
                                          SDValue Op,
                                          SelectionDAG *DAG) const {
assert(!OpInfo.Codes.empty() && "Must have at least one constraint")(static_cast <bool> (!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? void (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5014, __extension__ __PRETTY_FUNCTION__));

// Single-letter constraints ('r') are very common.
if (OpInfo.Codes.size() == 1) {
  OpInfo.ConstraintCode = OpInfo.Codes[0];
  OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
} else {
  ChooseConstraint(OpInfo, *this, Op, DAG);
}

// 'X' matches anything.
if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
  // Labels and constants are handled elsewhere ('X' is the only thing
  // that matches labels).  For Functions, the type here is the type of
  // the result, which is not what we want to look at; leave them alone.
  Value *v = OpInfo.CallOperandVal;
  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
    OpInfo.CallOperandVal = v;
    return;
  }

  if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
    return;

  // Otherwise, try to resolve it to something we know about by looking at
  // the actual operand type.
  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
    OpInfo.ConstraintCode = Repl;
    OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
  }
}
5045}

5047/// Given an exact SDIV by a constant, create a multiplication
5048/// with the multiplicative inverse of the constant.
5049static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
                            const SDLoc &dl, SelectionDAG &DAG,
                            SmallVectorImpl<SDNode *> &Created) {
SDValue Op0 = N->getOperand(0);
SDValue Op1 = N->getOperand(1);
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();

bool UseSRA = false;
SmallVector<SDValue, 16> Shifts, Factors;

auto BuildSDIVPattern = [&](ConstantSDNode *C) {
  if (C->isZero())
    return false;
  APInt Divisor = C->getAPIntValue();
  unsigned Shift = Divisor.countTrailingZeros();
  if (Shift) {
    Divisor.ashrInPlace(Shift);
    UseSRA = true;
  }
  // Calculate the multiplicative inverse, using Newton's method.
  APInt t;
  APInt Factor = Divisor;
  while ((t = Divisor * Factor) != 1)
    Factor *= APInt(Divisor.getBitWidth(), 2) - t;
  Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
  Factors.push_back(DAG.getConstant(Factor, dl, SVT));
  return true;
};

// Collect all magic values from the build vector.
if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
  return SDValue();

SDValue Shift, Factor;
if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
  Factor = DAG.getBuildVector(VT, dl, Factors);
} else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
  assert(Shifts.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))
         "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__))
         "vectors")(static_cast <bool> (Shifts.size() == 1 && Factors
.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("Shifts.size() == 1 && Factors.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5092, __extension__ __PRETTY_FUNCTION__));
  Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
  Factor = DAG.getSplatVector(VT, dl, Factors[0]);
} else {
  assert(isa<ConstantSDNode>(Op1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(Op1) &&
 "Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(Op1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5096, __extension__ __PRETTY_FUNCTION__));
  Shift = Shifts[0];
  Factor = Factors[0];
}

SDValue Res = Op0;

// Shift the value upfront if it is even, so the LSB is one.
if (UseSRA) {
  // TODO: For UDIV use SRL instead of SRA.
  SDNodeFlags Flags;
  Flags.setExact(true);
  Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
  Created.push_back(Res.getNode());
}

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5113}

5115SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
                            SelectionDAG &DAG,
                            SmallVectorImpl<SDNode *> &Created) const {
AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (TLI.isIntDivCheap(N->getValueType(0), Attr))
  return SDValue(N, 0); // Lower SDIV as SDIV
return SDValue();
5123}

5125/// Given an ISD::SDIV node expressing a divide by constant,
5126/// return a DAG expression to select that will generate the same value by
5127/// multiplying by a magic number.
5128/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5129SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
                                bool IsAfterLegalization,
                                SmallVectorImpl<SDNode *> &Created) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();
unsigned EltBits = VT.getScalarSizeInBits();
EVT MulVT;

// Check to see if we can do this.
// FIXME: We should be more aggressive here.
if (!isTypeLegal(VT)) {
  // Limit this to simple scalars for now.
  if (VT.isVector() || !VT.isSimple())
    return SDValue();

  // If this type will be promoted to a large enough type with a legal
  // multiply operation, we can go ahead and do this transform.
  if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
    return SDValue();

  MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
  if (MulVT.getSizeInBits() < (2 * EltBits) ||
      !isOperationLegal(ISD::MUL, MulVT))
    return SDValue();
}

// If the sdiv has an 'exact' bit we can use a simpler lowering.
if (N->getFlags().hasExact())
  return BuildExactSDIV(*this, N, dl, DAG, Created);

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

auto BuildSDIVPattern = [&](ConstantSDNode *C) {
  if (C->isZero())
    return false;

  const APInt &Divisor = C->getAPIntValue();
  SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(Divisor);
  int NumeratorFactor = 0;
  int ShiftMask = -1;

  if (Divisor.isOne() || Divisor.isAllOnes()) {
    // If d is +1/-1, we just multiply the numerator by +1/-1.
    NumeratorFactor = Divisor.getSExtValue();
    magics.Magic = 0;
    magics.ShiftAmount = 0;
    ShiftMask = 0;
  } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
    // If d > 0 and m < 0, add the numerator.
    NumeratorFactor = 1;
  } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
    // If d < 0 and m > 0, subtract the numerator.
    NumeratorFactor = -1;
  }

  MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
  Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
  Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
  ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
  return true;
};

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

// Collect the shifts / magic values from each element.
if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
  return SDValue();

SDValue MagicFactor, Factor, Shift, ShiftMask;
if (N1.getOpcode() == ISD::BUILD_VECTOR) {
  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
  Factor = DAG.getBuildVector(VT, dl, Factors);
  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
  ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
  assert(MagicFactors.size() == 1 && Factors.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
 Factors.size() == 1 && Shifts.size() == 1 &&
 ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
         Shifts.size() == 1 && ShiftMasks.size() == 1 &&(static_cast <bool> (MagicFactors.size() == 1 &&
 Factors.size() == 1 && Shifts.size() == 1 &&
 ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
         "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (MagicFactors.size() == 1 &&
 Factors.size() == 1 && Shifts.size() == 1 &&
 ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__))
         "vectors")(static_cast <bool> (MagicFactors.size() == 1 &&
 Factors.size() == 1 && Shifts.size() == 1 &&
 ShiftMasks.size() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("MagicFactors.size() == 1 && Factors.size() == 1 && Shifts.size() == 1 && ShiftMasks.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5211, __extension__ __PRETTY_FUNCTION__));
  MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
  Factor = DAG.getSplatVector(VT, dl, Factors[0]);
  Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
  ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
} else {
  assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
 "Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5217, __extension__ __PRETTY_FUNCTION__));
  MagicFactor = MagicFactors[0];
  Factor = Factors[0];
  Shift = Shifts[0];
  ShiftMask = ShiftMasks[0];
}

// Multiply the numerator (operand 0) by the magic value.
// FIXME: We should support doing a MUL in a wider type.
auto GetMULHS = [&](SDValue X, SDValue Y) {
  // If the type isn't legal, use a wider mul of the the type calculated
  // earlier.
  if (!isTypeLegal(VT)) {
    X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
    Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
    Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
    Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
                    DAG.getShiftAmountConstant(EltBits, MulVT, dl));
    return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
  }

  if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
    return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
  if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
    SDValue LoHi =
        DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
    return SDValue(LoHi.getNode(), 1);
  }
  return SDValue();
};

SDValue Q = GetMULHS(N0, MagicFactor);
if (!Q)
  return SDValue();

Created.push_back(Q.getNode());

// (Optionally) Add/subtract the numerator using Factor.
Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
Created.push_back(Factor.getNode());
Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
Created.push_back(Q.getNode());

// Shift right algebraic by shift value.
Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
Created.push_back(Q.getNode());

// Extract the sign bit, mask it and add it to the quotient.
SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
Created.push_back(T.getNode());
T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
Created.push_back(T.getNode());
return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5271}

5273/// Given an ISD::UDIV node expressing a divide by constant,
5274/// return a DAG expression to select that will generate the same value by
5275/// multiplying by a magic number.
5276/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
5277SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
                                bool IsAfterLegalization,
                                SmallVectorImpl<SDNode *> &Created) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();
unsigned EltBits = VT.getScalarSizeInBits();
EVT MulVT;

// Check to see if we can do this.
// FIXME: We should be more aggressive here.
if (!isTypeLegal(VT)) {
  // Limit this to simple scalars for now.
  if (VT.isVector() || !VT.isSimple())
    return SDValue();

  // If this type will be promoted to a large enough type with a legal
  // multiply operation, we can go ahead and do this transform.
  if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
    return SDValue();

  MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
  if (MulVT.getSizeInBits() < (2 * EltBits) ||
      !isOperationLegal(ISD::MUL, MulVT))
    return SDValue();
}

bool UseNPQ = false;
SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

auto BuildUDIVPattern = [&](ConstantSDNode *C) {
  if (C->isZero())
    return false;
  // FIXME: We should use a narrower constant when the upper
  // bits are known to be zero.
  const APInt& Divisor = C->getAPIntValue();
  UnsignedDivisonByConstantInfo magics = UnsignedDivisonByConstantInfo::get(Divisor);
  unsigned PreShift = 0, PostShift = 0;

  // If the divisor is even, we can avoid using the expensive fixup by
  // shifting the divided value upfront.
  if (magics.IsAdd != 0 && !Divisor[0]) {
    PreShift = Divisor.countTrailingZeros();
    // Get magic number for the shifted divisor.
    magics = UnsignedDivisonByConstantInfo::get(Divisor.lshr(PreShift), PreShift);
    assert(magics.IsAdd == 0 && "Should use cheap fixup now")(static_cast <bool> (magics.IsAdd == 0 && "Should use cheap fixup now"
) ? void (0) : __assert_fail ("magics.IsAdd == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5324, __extension__ __PRETTY_FUNCTION__));
  }

  APInt Magic = magics.Magic;

  unsigned SelNPQ;
  if (magics.IsAdd == 0 || Divisor.isOne()) {
    assert(magics.ShiftAmount < Divisor.getBitWidth() &&(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
 (0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__))
           "We shouldn't generate an undefined shift!")(static_cast <bool> (magics.ShiftAmount < Divisor.getBitWidth
() && "We shouldn't generate an undefined shift!") ? void
 (0) : __assert_fail ("magics.ShiftAmount < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5332, __extension__ __PRETTY_FUNCTION__));
    PostShift = magics.ShiftAmount;
    SelNPQ = false;
  } else {
    PostShift = magics.ShiftAmount - 1;
    SelNPQ = true;
  }

  PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
  MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
  NPQFactors.push_back(
      DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
                             : APInt::getZero(EltBits),
                      dl, SVT));
  PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
  UseNPQ |= SelNPQ;
  return true;
};

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

// Collect the shifts/magic values from each element.
if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
  return SDValue();

SDValue PreShift, PostShift, MagicFactor, NPQFactor;
if (N1.getOpcode() == ISD::BUILD_VECTOR) {
  PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
  NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
  PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
} else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
  assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))
         NPQFactors.size() == 1 && PostShifts.size() == 1 &&(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__))
         "Expected matchUnaryPredicate to return one for scalable vectors")(static_cast <bool> (PreShifts.size() == 1 && MagicFactors
.size() == 1 && NPQFactors.size() == 1 && PostShifts
.size() == 1 && "Expected matchUnaryPredicate to return one for scalable vectors"
) ? void (0) : __assert_fail ("PreShifts.size() == 1 && MagicFactors.size() == 1 && NPQFactors.size() == 1 && PostShifts.size() == 1 && \"Expected matchUnaryPredicate to return one for scalable vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5367, __extension__ __PRETTY_FUNCTION__));
  PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
  MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
  NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
  PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
} else {
  assert(isa<ConstantSDNode>(N1) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(N1) &&
 "Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(N1) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5373, __extension__ __PRETTY_FUNCTION__));
  PreShift = PreShifts[0];
  MagicFactor = MagicFactors[0];
  PostShift = PostShifts[0];
}

SDValue Q = N0;
Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
Created.push_back(Q.getNode());

// FIXME: We should support doing a MUL in a wider type.
auto GetMULHU = [&](SDValue X, SDValue Y) {
  // If the type isn't legal, use a wider mul of the the type calculated
  // earlier.
  if (!isTypeLegal(VT)) {
    X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
    Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
    Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
    Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
                    DAG.getShiftAmountConstant(EltBits, MulVT, dl));
    return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
  }

  if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
    return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
  if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
    SDValue LoHi =
        DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
    return SDValue(LoHi.getNode(), 1);
  }
  return SDValue(); // No mulhu or equivalent
};

// Multiply the numerator (operand 0) by the magic value.
Q = GetMULHU(Q, MagicFactor);
if (!Q)
  return SDValue();

Created.push_back(Q.getNode());

if (UseNPQ) {
  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
  Created.push_back(NPQ.getNode());

  // For vectors we might have a mix of non-NPQ/NPQ paths, so use
  // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
  if (VT.isVector())
    NPQ = GetMULHU(NPQ, NPQFactor);
  else
    NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

  Created.push_back(NPQ.getNode());

  Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
  Created.push_back(Q.getNode());
}

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
Created.push_back(Q.getNode());

EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

SDValue One = DAG.getConstant(1, dl, VT);
SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
return DAG.getSelect(dl, VT, IsOne, N0, Q);
5438}

5440/// If all values in Values that *don't* match the predicate are same 'splat'
5441/// value, then replace all values with that splat value.
5442/// Else, if AlternativeReplacement was provided, then replace all values that
5443/// do match predicate with AlternativeReplacement value.
5444static void
5445turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
                        std::function<bool(SDValue)> Predicate,
                        SDValue AlternativeReplacement = SDValue()) {
SDValue Replacement;
// Is there a value for which the Predicate does *NOT* match? What is it?
auto SplatValue = llvm::find_if_not(Values, Predicate);
if (SplatValue != Values.end()) {
  // Does Values consist only of SplatValue's and values matching Predicate?
  if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
        return Value == *SplatValue || Predicate(Value);
      })) // Then we shall replace values matching predicate with SplatValue.
    Replacement = *SplatValue;
}
if (!Replacement) {
  // Oops, we did not find the "baseline" splat value.
  if (!AlternativeReplacement)
    return; // Nothing to do.
  // Let's replace with provided value then.
  Replacement = AlternativeReplacement;
}
std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5466}

5468/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5469/// where the divisor is constant and the comparison target is zero,
5470/// return a DAG expression that will generate the same comparison result
5471/// using only multiplications, additions and shifts/rotations.
5472/// Ref: "Hacker's Delight" 10-17.
5473SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
                                      SDValue CompTargetNode,
                                      ISD::CondCode Cond,
                                      DAGCombinerInfo &DCI,
                                      const SDLoc &DL) const {
SmallVector<SDNode *, 5> Built;
if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
                                       DCI, DL, Built)) {
  for (SDNode *N : Built)
    DCI.AddToWorklist(N);
  return Folded;
}

return SDValue();
5487}

5489SDValue
5490TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
                                SDValue CompTargetNode, ISD::CondCode Cond,
                                DAGCombinerInfo &DCI, const SDLoc &DL,
                                SmallVectorImpl<SDNode *> &Created) const {
// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
// - D must be constant, with D = D0 * 2^K where D0 is odd
// - P is the multiplicative inverse of D0 modulo 2^W
// - Q = floor(((2^W) - 1) / D)
// where W is the width of the common type of N and D.
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __extension__ __PRETTY_FUNCTION__))
       "Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5500, __extension__ __PRETTY_FUNCTION__));

SelectionDAG &DAG = DCI.DAG;

EVT VT = REMNode.getValueType();
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
EVT ShSVT = ShVT.getScalarType();

// If MUL is unavailable, we cannot proceed in any case.
if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
  return SDValue();

bool ComparingWithAllZeros = true;
bool AllComparisonsWithNonZerosAreTautological = true;
bool HadTautologicalLanes = false;
bool AllLanesAreTautological = true;
bool HadEvenDivisor = false;
bool AllDivisorsArePowerOfTwo = true;
bool HadTautologicalInvertedLanes = false;
SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;

auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
  // Division by 0 is UB. Leave it to be constant-folded elsewhere.
  if (CDiv->isZero())
    return false;

  const APInt &D = CDiv->getAPIntValue();
  const APInt &Cmp = CCmp->getAPIntValue();

  ComparingWithAllZeros &= Cmp.isZero();

  // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
  // if C2 is not less than C1, the comparison is always false.
  // But we will only be able to produce the comparison that will give the
  // opposive tautological answer. So this lane would need to be fixed up.
  bool TautologicalInvertedLane = D.ule(Cmp);
  HadTautologicalInvertedLanes |= TautologicalInvertedLane;

  // If all lanes are tautological (either all divisors are ones, or divisor
  // is not greater than the constant we are comparing with),
  // we will prefer to avoid the fold.
  bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
  HadTautologicalLanes |= TautologicalLane;
  AllLanesAreTautological &= TautologicalLane;

  // If we are comparing with non-zero, we need'll need  to subtract said
  // comparison value from the LHS. But there is no point in doing that if
  // every lane where we are comparing with non-zero is tautological..
  if (!Cmp.isZero())
    AllComparisonsWithNonZerosAreTautological &= TautologicalLane;

  // Decompose D into D0 * 2^K
  unsigned K = D.countTrailingZeros();
  assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOne() || (K == 0)) &&
 "For divisor '1' we won't rotate.") ? void (0) : __assert_fail
 ("(!D.isOne() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5554, __extension__ __PRETTY_FUNCTION__));
  APInt D0 = D.lshr(K);

  // D is even if it has trailing zeros.
  HadEvenDivisor |= (K != 0);
  // D is a power-of-two if D0 is one.
  // If all divisors are power-of-two, we will prefer to avoid the fold.
  AllDivisorsArePowerOfTwo &= D0.isOne();

  // P = inv(D0, 2^W)
  // 2^W requires W + 1 bits, so we have to extend and then truncate.
  unsigned W = D.getBitWidth();
  APInt P = D0.zext(W + 1)
                .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
                .trunc(W);
  assert(!P.isZero() && "No multiplicative inverse!")(static_cast <bool> (!P.isZero() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isZero() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5569, __extension__ __PRETTY_FUNCTION__)); // unreachable
  assert((D0 * P).isOne() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOne() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOne() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5570, __extension__ __PRETTY_FUNCTION__));

  // Q = floor((2^W - 1) u/ D)
  // R = ((2^W - 1) u% D)
  APInt Q, R;
  APInt::udivrem(APInt::getAllOnes(W), D, Q, R);

  // If we are comparing with zero, then that comparison constant is okay,
  // else it may need to be one less than that.
  if (Cmp.ugt(R))
    Q -= 1;

  assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5583, __extension__ __PRETTY_FUNCTION__))
         "We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5583, __extension__ __PRETTY_FUNCTION__));

  // If the lane is tautological the result can be constant-folded.
  if (TautologicalLane) {
    // Set P and K amount to a bogus values so we can try to splat them.
    P = 0;
    K = -1;
    // And ensure that comparison constant is tautological,
    // it will always compare true/false.
    Q = -1;
  }

  PAmts.push_back(DAG.getConstant(P, DL, SVT));
  KAmts.push_back(
      DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
  QAmts.push_back(DAG.getConstant(Q, DL, SVT));
  return true;
};

SDValue N = REMNode.getOperand(0);
SDValue D = REMNode.getOperand(1);

// Collect the values from each element.
if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
  return SDValue();

// If all lanes are tautological, the result can be constant-folded.
if (AllLanesAreTautological)
  return SDValue();

// If this is a urem by a powers-of-two, avoid the fold since it can be
// best implemented as a bit test.
if (AllDivisorsArePowerOfTwo)
  return SDValue();

SDValue PVal, KVal, QVal;
if (D.getOpcode() == ISD::BUILD_VECTOR) {
  if (HadTautologicalLanes) {
    // Try to turn PAmts into a splat, since we don't care about the values
    // that are currently '0'. If we can't, just keep '0'`s.
    turnVectorIntoSplatVector(PAmts, isNullConstant);
    // Try to turn KAmts into a splat, since we don't care about the values
    // that are currently '-1'. If we can't, change them to '0'`s.
    turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
                              DAG.getConstant(0, DL, ShSVT));
  }

  PVal = DAG.getBuildVector(VT, DL, PAmts);
  KVal = DAG.getBuildVector(ShVT, DL, KAmts);
  QVal = DAG.getBuildVector(VT, DL, QAmts);
} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
  assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
 "SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__))
         "Expected matchBinaryPredicate to return one element for "(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
 "SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__))
         "SPLAT_VECTORs")(static_cast <bool> (PAmts.size() == 1 && KAmts
.size() == 1 && QAmts.size() == 1 && "Expected matchBinaryPredicate to return one element for "
 "SPLAT_VECTORs") ? void (0) : __assert_fail ("PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchBinaryPredicate to return one element for \" \"SPLAT_VECTORs\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5636, __extension__ __PRETTY_FUNCTION__));
  PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
  KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
  QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
} else {
  PVal = PAmts[0];
  KVal = KAmts[0];
  QVal = QAmts[0];
}

if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
    return SDValue(); // FIXME: Could/should use `ISD::ADD`?
  assert(CompTargetNode.getValueType() == N.getValueType() &&(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5650, __extension__ __PRETTY_FUNCTION__))
         "Expecting that the types on LHS and RHS of comparisons match.")(static_cast <bool> (CompTargetNode.getValueType() == N
.getValueType() && "Expecting that the types on LHS and RHS of comparisons match."
) ? void (0) : __assert_fail ("CompTargetNode.getValueType() == N.getValueType() && \"Expecting that the types on LHS and RHS of comparisons match.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5650, __extension__ __PRETTY_FUNCTION__));
  N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
}

// (mul N, P)
SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
Created.push_back(Op0.getNode());

// Rotate right only if any divisor was even. We avoid rotates for all-odd
// divisors as a performance improvement, since rotating by 0 is a no-op.
if (HadEvenDivisor) {
  // We need ROTR to do this.
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
    return SDValue();
  // UREM: (rotr (mul N, P), K)
  Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
  Created.push_back(Op0.getNode());
}

// UREM: (setule/setugt (rotr (mul N, P), K), Q)
SDValue NewCC =
    DAG.getSetCC(DL, SETCCVT, Op0, QVal,
                 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
if (!HadTautologicalInvertedLanes)
  return NewCC;

// If any lanes previously compared always-false, the NewCC will give
// always-true result for them, so we need to fixup those lanes.
// Or the other way around for inequality predicate.
assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5679, __extension__ __PRETTY_FUNCTION__));
Created.push_back(NewCC.getNode());

// x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
// if C2 is not less than C1, the comparison is always false.
// But we have produced the comparison that will give the
// opposive tautological answer. So these lanes would need to be fixed up.
SDValue TautologicalInvertedChannels =
    DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
Created.push_back(TautologicalInvertedChannels.getNode());

// NOTE: we avoid letting illegal types through even if we're before legalize
// ops – legalization has a hard time producing good code for this.
if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
  // If we have a vector select, let's replace the comparison results in the
  // affected lanes with the correct tautological result.
  SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
                                            DL, SETCCVT, SETCCVT);
  return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
                     Replacement, NewCC);
}

// Else, we can just invert the comparison result in the appropriate lanes.
//
// NOTE: see the note above VSELECT above.
if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
  return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
                     TautologicalInvertedChannels);

return SDValue(); // Don't know how to lower.
5709}

5711/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5712/// where the divisor is constant and the comparison target is zero,
5713/// return a DAG expression that will generate the same comparison result
5714/// using only multiplications, additions and shifts/rotations.
5715/// Ref: "Hacker's Delight" 10-17.
5716SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
                                      SDValue CompTargetNode,
                                      ISD::CondCode Cond,
                                      DAGCombinerInfo &DCI,
                                      const SDLoc &DL) const {
SmallVector<SDNode *, 7> Built;
if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
                                       DCI, DL, Built)) {
  assert(Built.size() <= 7 && "Max size prediction failed.")(static_cast <bool> (Built.size() <= 7 && "Max size prediction failed."
) ? void (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5724, __extension__ __PRETTY_FUNCTION__));
  for (SDNode *N : Built)
    DCI.AddToWorklist(N);
  return Folded;
}

return SDValue();
5731}

5733SDValue
5734TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
                                SDValue CompTargetNode, ISD::CondCode Cond,
                                DAGCombinerInfo &DCI, const SDLoc &DL,
                                SmallVectorImpl<SDNode *> &Created) const {
// Fold:
//   (seteq/ne (srem N, D), 0)
// To:
//   (setule/ugt (rotr (add (mul N, P), A), K), Q)
//
// - D must be constant, with D = D0 * 2^K where D0 is odd
// - P is the multiplicative inverse of D0 modulo 2^W
// - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
// - Q = floor((2 * A) / (2^K))
// where W is the width of the common type of N and D.
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__))
       "Only applicable for (in)equality comparisons.")(static_cast <bool> ((Cond == ISD::SETEQ || Cond == ISD
::SETNE) && "Only applicable for (in)equality comparisons."
) ? void (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5749, __extension__ __PRETTY_FUNCTION__));

SelectionDAG &DAG = DCI.DAG;

EVT VT = REMNode.getValueType();
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
EVT ShSVT = ShVT.getScalarType();

// If we are after ops legalization, and MUL is unavailable, we can not
// proceed.
if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
  return SDValue();

// TODO: Could support comparing with non-zero too.
ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
if (!CompTarget || !CompTarget->isZero())
  return SDValue();

bool HadIntMinDivisor = false;
bool HadOneDivisor = false;
bool AllDivisorsAreOnes = true;
bool HadEvenDivisor = false;
bool NeedToApplyOffset = false;
bool AllDivisorsArePowerOfTwo = true;
SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;

auto BuildSREMPattern = [&](ConstantSDNode *C) {
  // Division by 0 is UB. Leave it to be constant-folded elsewhere.
  if (C->isZero())
    return false;

  // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.

  // WARNING: this fold is only valid for positive divisors!
  APInt D = C->getAPIntValue();
  if (D.isNegative())
    D.negate(); //  `rem %X, -C` is equivalent to `rem %X, C`

  HadIntMinDivisor |= D.isMinSignedValue();

  // If all divisors are ones, we will prefer to avoid the fold.
  HadOneDivisor |= D.isOne();
  AllDivisorsAreOnes &= D.isOne();

  // Decompose D into D0 * 2^K
  unsigned K = D.countTrailingZeros();
  assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.")(static_cast <bool> ((!D.isOne() || (K == 0)) &&
 "For divisor '1' we won't rotate.") ? void (0) : __assert_fail
 ("(!D.isOne() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5796, __extension__ __PRETTY_FUNCTION__));
  APInt D0 = D.lshr(K);

  if (!D.isMinSignedValue()) {
    // D is even if it has trailing zeros; unless it's INT_MIN, in which case
    // we don't care about this lane in this fold, we'll special-handle it.
    HadEvenDivisor |= (K != 0);
  }

  // D is a power-of-two if D0 is one. This includes INT_MIN.
  // If all divisors are power-of-two, we will prefer to avoid the fold.
  AllDivisorsArePowerOfTwo &= D0.isOne();

  // P = inv(D0, 2^W)
  // 2^W requires W + 1 bits, so we have to extend and then truncate.
  unsigned W = D.getBitWidth();
  APInt P = D0.zext(W + 1)
                .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
                .trunc(W);
  assert(!P.isZero() && "No multiplicative inverse!")(static_cast <bool> (!P.isZero() && "No multiplicative inverse!"
) ? void (0) : __assert_fail ("!P.isZero() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5815, __extension__ __PRETTY_FUNCTION__)); // unreachable
  assert((D0 * P).isOne() && "Multiplicative inverse sanity check.")(static_cast <bool> ((D0 * P).isOne() && "Multiplicative inverse sanity check."
) ? void (0) : __assert_fail ("(D0 * P).isOne() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5816, __extension__ __PRETTY_FUNCTION__));

  // A = floor((2^(W - 1) - 1) / D0) & -2^K
  APInt A = APInt::getSignedMaxValue(W).udiv(D0);
  A.clearLowBits(K);

  if (!D.isMinSignedValue()) {
    // If divisor INT_MIN, then we don't care about this lane in this fold,
    // we'll special-handle it.
    NeedToApplyOffset |= A != 0;
  }

  // Q = floor((2 * A) / (2^K))
  APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));

  assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&(static_cast <bool> (APInt::getAllOnes(SVT.getSizeInBits
()).ugt(A) && "We are expecting that A is always less than all-ones for SVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5832, __extension__ __PRETTY_FUNCTION__))
         "We are expecting that A is always less than all-ones for SVT")(static_cast <bool> (APInt::getAllOnes(SVT.getSizeInBits
()).ugt(A) && "We are expecting that A is always less than all-ones for SVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5832, __extension__ __PRETTY_FUNCTION__));
  assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5834, __extension__ __PRETTY_FUNCTION__))
         "We are expecting that K is always less than all-ones for ShSVT")(static_cast <bool> (APInt::getAllOnes(ShSVT.getSizeInBits
()).ugt(K) && "We are expecting that K is always less than all-ones for ShSVT"
) ? void (0) : __assert_fail ("APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5834, __extension__ __PRETTY_FUNCTION__));

  // If the divisor is 1 the result can be constant-folded. Likewise, we
  // don't care about INT_MIN lanes, those can be set to undef if appropriate.
  if (D.isOne()) {
    // Set P, A and K to a bogus values so we can try to splat them.
    P = 0;
    A = -1;
    K = -1;

    // x ?% 1 == 0  <-->  true  <-->  x u<= -1
    Q = -1;
  }

  PAmts.push_back(DAG.getConstant(P, DL, SVT));
  AAmts.push_back(DAG.getConstant(A, DL, SVT));
  KAmts.push_back(
      DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
  QAmts.push_back(DAG.getConstant(Q, DL, SVT));
  return true;
};

SDValue N = REMNode.getOperand(0);
SDValue D = REMNode.getOperand(1);

// Collect the values from each element.
if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
  return SDValue();

// If this is a srem by a one, avoid the fold since it can be constant-folded.
if (AllDivisorsAreOnes)
  return SDValue();

// If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
// since it can be best implemented as a bit test.
if (AllDivisorsArePowerOfTwo)
  return SDValue();

SDValue PVal, AVal, KVal, QVal;
if (D.getOpcode() == ISD::BUILD_VECTOR) {
  if (HadOneDivisor) {
    // Try to turn PAmts into a splat, since we don't care about the values
    // that are currently '0'. If we can't, just keep '0'`s.
    turnVectorIntoSplatVector(PAmts, isNullConstant);
    // Try to turn AAmts into a splat, since we don't care about the
    // values that are currently '-1'. If we can't, change them to '0'`s.
    turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
                              DAG.getConstant(0, DL, SVT));
    // Try to turn KAmts into a splat, since we don't care about the values
    // that are currently '-1'. If we can't, change them to '0'`s.
    turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
                              DAG.getConstant(0, DL, ShSVT));
  }

  PVal = DAG.getBuildVector(VT, DL, PAmts);
  AVal = DAG.getBuildVector(VT, DL, AAmts);
  KVal = DAG.getBuildVector(ShVT, DL, KAmts);
  QVal = DAG.getBuildVector(VT, DL, QAmts);
} else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
  assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__))
         QAmts.size() == 1 &&(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__))
         "Expected matchUnaryPredicate to return one element for scalable "(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__))
         "vectors")(static_cast <bool> (PAmts.size() == 1 && AAmts
.size() == 1 && KAmts.size() == 1 && QAmts.size
() == 1 && "Expected matchUnaryPredicate to return one element for scalable "
 "vectors") ? void (0) : __assert_fail ("PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 && \"Expected matchUnaryPredicate to return one element for scalable \" \"vectors\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5896, __extension__ __PRETTY_FUNCTION__));
  PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
  AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
  KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
  QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
} else {
  assert(isa<ConstantSDNode>(D) && "Expected a constant")(static_cast <bool> (isa<ConstantSDNode>(D) &&
 "Expected a constant") ? void (0) : __assert_fail ("isa<ConstantSDNode>(D) && \"Expected a constant\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5902, __extension__ __PRETTY_FUNCTION__));
  PVal = PAmts[0];
  AVal = AAmts[0];
  KVal = KAmts[0];
  QVal = QAmts[0];
}

// (mul N, P)
SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
Created.push_back(Op0.getNode());

if (NeedToApplyOffset) {
  // We need ADD to do this.
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
    return SDValue();

  // (add (mul N, P), A)
  Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
  Created.push_back(Op0.getNode());
}

// Rotate right only if any divisor was even. We avoid rotates for all-odd
// divisors as a performance improvement, since rotating by 0 is a no-op.
if (HadEvenDivisor) {
  // We need ROTR to do this.
  if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
    return SDValue();
  // SREM: (rotr (add (mul N, P), A), K)
  Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal);
  Created.push_back(Op0.getNode());
}

// SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
SDValue Fold =
    DAG.getSetCC(DL, SETCCVT, Op0, QVal,
                 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

// If we didn't have lanes with INT_MIN divisor, then we're done.
if (!HadIntMinDivisor)
  return Fold;

// That fold is only valid for positive divisors. Which effectively means,
// it is invalid for INT_MIN divisors. So if we have such a lane,
// we must fix-up results for said lanes.
assert(VT.isVector() && "Can/should only get here for vectors.")(static_cast <bool> (VT.isVector() && "Can/should only get here for vectors."
) ? void (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5946, __extension__ __PRETTY_FUNCTION__));

// NOTE: we avoid letting illegal types through even if we're before legalize
// ops – legalization has a hard time producing good code for the code that
// follows.
if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
    !isOperationLegalOrCustom(ISD::AND, VT) ||
    !isOperationLegalOrCustom(Cond, VT) ||
    !isOperationLegalOrCustom(ISD::VSELECT, SETCCVT))
  return SDValue();

Created.push_back(Fold.getNode());

SDValue IntMin = DAG.getConstant(
    APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
SDValue IntMax = DAG.getConstant(
    APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
SDValue Zero =
    DAG.getConstant(APInt::getZero(SVT.getScalarSizeInBits()), DL, VT);

// Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
Created.push_back(DivisorIsIntMin.getNode());

// (N s% INT_MIN) ==/!= 0  <-->  (N & INT_MAX) ==/!= 0
SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
Created.push_back(Masked.getNode());
SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
Created.push_back(MaskedIsZero.getNode());

// To produce final result we need to blend 2 vectors: 'SetCC' and
// 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
// from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
// constant-folded, select can get lowered to a shuffle with constant mask.
SDValue Blended = DAG.getNode(ISD::VSELECT, DL, SETCCVT, DivisorIsIntMin,
                              MaskedIsZero, Fold);

return Blended;
5984}

5986bool TargetLowering::
5987verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
if (!isa<ConstantSDNode>(Op.getOperand(0))) {
  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
                              "be a constant integer");
  return true;
}

return false;
5995}

5997SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
                                       const DenormalMode &Mode) const {
SDLoc DL(Op);
EVT VT = Op.getValueType();
EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
// Testing it with denormal inputs to avoid wrong estimate.
if (Mode.Input == DenormalMode::IEEE) {
  // This is specifically a check for the handling of denormal inputs,
  // not the result.

  // Test = fabs(X) < SmallestNormal
  const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
  APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
  SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
  SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
  return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
}
// Test = X == 0.0
return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
6017}

6019SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
                                           bool LegalOps, bool OptForSize,
                                           NegatibleCost &Cost,
                                           unsigned Depth) const {
// fneg is removable even if it has multiple uses.
if (Op.getOpcode() == ISD::FNEG) {
  Cost = NegatibleCost::Cheaper;
  return Op.getOperand(0);
}

// Don't recurse exponentially.
if (Depth > SelectionDAG::MaxRecursionDepth)
  return SDValue();

// Pre-increment recursion depth for use in recursive calls.
++Depth;
const SDNodeFlags Flags = Op->getFlags();
const TargetOptions &Options = DAG.getTarget().Options;
EVT VT = Op.getValueType();
unsigned Opcode = Op.getOpcode();

// Don't allow anything with multiple uses unless we know it is free.
if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
  bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
                      isFPExtFree(VT, Op.getOperand(0).getValueType());
  if (!IsFreeExtend)
    return SDValue();
}

auto RemoveDeadNode = [&](SDValue N) {
  if (N && N.getNode()->use_empty())
    DAG.RemoveDeadNode(N.getNode());
};

SDLoc DL(Op);

// Because getNegatedExpression can delete nodes we need a handle to keep
// temporary nodes alive in case the recursion manages to create an identical
// node.
std::list<HandleSDNode> Handles;

switch (Opcode) {
case ISD::ConstantFP: {
  // Don't invert constant FP values after legalization unless the target says
  // the negated constant is legal.
  bool IsOpLegal =
      isOperationLegal(ISD::ConstantFP, VT) ||
      isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
                   OptForSize);

  if (LegalOps && !IsOpLegal)
    break;

  APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
  V.changeSign();
  SDValue CFP = DAG.getConstantFP(V, DL, VT);

  // If we already have the use of the negated floating constant, it is free
  // to negate it even it has multiple uses.
  if (!Op.hasOneUse() && CFP.use_empty())
    break;
  Cost = NegatibleCost::Neutral;
  return CFP;
}
case ISD::BUILD_VECTOR: {
  // Only permit BUILD_VECTOR of constants.
  if (llvm::any_of(Op->op_values(), [&](SDValue N) {
        return !N.isUndef() && !isa<ConstantFPSDNode>(N);
      }))
    break;

  bool IsOpLegal =
      (isOperationLegal(ISD::ConstantFP, VT) &&
       isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
      llvm::all_of(Op->op_values(), [&](SDValue N) {
        return N.isUndef() ||
               isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
                            OptForSize);
      });

  if (LegalOps && !IsOpLegal)
    break;

  SmallVector<SDValue, 4> Ops;
  for (SDValue C : Op->op_values()) {
    if (C.isUndef()) {
      Ops.push_back(C);
      continue;
    }
    APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
    V.changeSign();
    Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
  }
  Cost = NegatibleCost::Neutral;
  return DAG.getBuildVector(VT, DL, Ops);
}
case ISD::FADD: {
  if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
    break;

  // After operation legalization, it might not be legal to create new FSUBs.
  if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
    break;
  SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

  // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
  NegatibleCost CostX = NegatibleCost::Expensive;
  SDValue NegX =
      getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
  // Prevent this node from being deleted by the next call.
  if (NegX)
    Handles.emplace_back(NegX);

  // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
  NegatibleCost CostY = NegatibleCost::Expensive;
  SDValue NegY =
      getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

  // We're done with the handles.
  Handles.clear();

  // Negate the X if its cost is less or equal than Y.
  if (NegX && (CostX <= CostY)) {
    Cost = CostX;
    SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
    if (NegY != N)
      RemoveDeadNode(NegY);
    return N;
  }

  // Negate the Y if it is not expensive.
  if (NegY) {
    Cost = CostY;
    SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
    if (NegX != N)
      RemoveDeadNode(NegX);
    return N;
  }
  break;
}
case ISD::FSUB: {
  // We can't turn -(A-B) into B-A when we honor signed zeros.
  if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
    break;

  SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
  // fold (fneg (fsub 0, Y)) -> Y
  if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
    if (C->isZero()) {
      Cost = NegatibleCost::Cheaper;
      return Y;
    }

  // fold (fneg (fsub X, Y)) -> (fsub Y, X)
  Cost = NegatibleCost::Neutral;
  return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
}
case ISD::FMUL:
case ISD::FDIV: {
  SDValue X = Op.getOperand(0), Y = Op.getOperand(1);

  // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
  NegatibleCost CostX = NegatibleCost::Expensive;
  SDValue NegX =
      getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
  // Prevent this node from being deleted by the next call.
  if (NegX)
    Handles.emplace_back(NegX);

  // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
  NegatibleCost CostY = NegatibleCost::Expensive;
  SDValue NegY =
      getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

  // We're done with the handles.
  Handles.clear();

  // Negate the X if its cost is less or equal than Y.
  if (NegX && (CostX <= CostY)) {
    Cost = CostX;
    SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
    if (NegY != N)
      RemoveDeadNode(NegY);
    return N;
  }

  // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
  if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
    if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
      break;

  // Negate the Y if it is not expensive.
  if (NegY) {
    Cost = CostY;
    SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
    if (NegX != N)
      RemoveDeadNode(NegX);
    return N;
  }
  break;
}
case ISD::FMA:
case ISD::FMAD: {
  if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
    break;

  SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
  NegatibleCost CostZ = NegatibleCost::Expensive;
  SDValue NegZ =
      getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
  // Give up if fail to negate the Z.
  if (!NegZ)
    break;

  // Prevent this node from being deleted by the next two calls.
  Handles.emplace_back(NegZ);

  // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
  NegatibleCost CostX = NegatibleCost::Expensive;
  SDValue NegX =
      getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
  // Prevent this node from being deleted by the next call.
  if (NegX)
    Handles.emplace_back(NegX);

  // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
  NegatibleCost CostY = NegatibleCost::Expensive;
  SDValue NegY =
      getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);

  // We're done with the handles.
  Handles.clear();

  // Negate the X if its cost is less or equal than Y.
  if (NegX && (CostX <= CostY)) {
    Cost = std::min(CostX, CostZ);
    SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
    if (NegY != N)
      RemoveDeadNode(NegY);
    return N;
  }

  // Negate the Y if it is not expensive.
  if (NegY) {
    Cost = std::min(CostY, CostZ);
    SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
    if (NegX != N)
      RemoveDeadNode(NegX);
    return N;
  }
  break;
}

case ISD::FP_EXTEND:
case ISD::FSIN:
  if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
                                          OptForSize, Cost, Depth))
    return DAG.getNode(Opcode, DL, VT, NegV);
  break;
case ISD::FP_ROUND:
  if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
                                          OptForSize, Cost, Depth))
    return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
  break;
}

return SDValue();
6286}

6288//===----------------------------------------------------------------------===//
6289// Legalization Utilities
6290//===----------------------------------------------------------------------===//

6292bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
                                  SDValue LHS, SDValue RHS,
                                  SmallVectorImpl<SDValue> &Result,
                                  EVT HiLoVT, SelectionDAG &DAG,
                                  MulExpansionKind Kind, SDValue LL,
                                  SDValue LH, SDValue RL, SDValue RH) const {
assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||(static_cast <bool> (Opcode == ISD::MUL || Opcode == ISD
::UMUL_LOHI || Opcode == ISD::SMUL_LOHI) ? void (0) : __assert_fail
 ("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6299, __extension__ __PRETTY_FUNCTION__))
       Opcode == ISD::SMUL_LOHI)(static_cast <bool> (Opcode == ISD::MUL || Opcode == ISD
::UMUL_LOHI || Opcode == ISD::SMUL_LOHI) ? void (0) : __assert_fail
 ("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6299, __extension__ __PRETTY_FUNCTION__));

bool HasMULHS = (Kind == MulExpansionKind::Always) ||
                isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
bool HasMULHU = (Kind == MulExpansionKind::Always) ||
                isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
                    isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
                    isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);

if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
  return false;

unsigned OuterBitSize = VT.getScalarSizeInBits();
unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();

// LL, LH, RL, and RH must be either all NULL or all set to a value.
assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||(static_cast <bool> ((LL.getNode() && LH.getNode
() && RL.getNode() && RH.getNode()) || (!LL.getNode
() && !LH.getNode() && !RL.getNode() &&
 !RH.getNode())) ? void (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6318, __extension__ __PRETTY_FUNCTION__))
       (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()))(static_cast <bool> ((LL.getNode() && LH.getNode
() && RL.getNode() && RH.getNode()) || (!LL.getNode
() && !LH.getNode() && !RL.getNode() &&
 !RH.getNode())) ? void (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6318, __extension__ __PRETTY_FUNCTION__));

SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
                        bool Signed) -> bool {
  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
    Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
    Hi = SDValue(Lo.getNode(), 1);
    return true;
  }
  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
    Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
    Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
    return true;
  }
  return false;
};

SDValue Lo, Hi;

if (!LL.getNode() && !RL.getNode() &&
    isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
}

if (!LL.getNode())
  return false;

APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
if (DAG.MaskedValueIsZero(LHS, HighMask) &&
    DAG.MaskedValueIsZero(RHS, HighMask)) {
  // The inputs are both zero-extended.
  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
    Result.push_back(Lo);
    Result.push_back(Hi);
    if (Opcode != ISD::MUL) {
      SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
      Result.push_back(Zero);
      Result.push_back(Zero);
    }
    return true;
  }
}

if (!VT.isVector() && Opcode == ISD::MUL &&
    DAG.ComputeNumSignBits(LHS) > InnerBitSize &&
    DAG.ComputeNumSignBits(RHS) > InnerBitSize) {
  // The input values are both sign-extended.
  // TODO non-MUL case?
  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
    Result.push_back(Lo);
    Result.push_back(Hi);
    return true;
  }
}

unsigned ShiftAmount = OuterBitSize - InnerBitSize;
EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
  // FIXME getShiftAmountTy does not always return a sensible result when VT
  // is an illegal type, and so the type may be too small to fit the shift
  // amount. Override it with i32. The shift will have to be legalized.
  ShiftAmountTy = MVT::i32;
}
SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);

if (!LH.getNode() && !RH.getNode() &&
    isOperationLegalOrCustom(ISD::SRL, VT) &&
    isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
}

if (!LH.getNode())
  return false;

if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
  return false;

Result.push_back(Lo);

if (Opcode == ISD::MUL) {
  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
  Result.push_back(Hi);
  return true;
}

// Compute the full width result.
auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
};

SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
  return false;

// This is effectively the add part of a multiply-add of half-sized operands,
// so it cannot overflow.
Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
  return false;

SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
                isOperationLegalOrCustom(ISD::ADDE, VT));
if (UseGlue)
  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
                     Merge(Lo, Hi));
else
  Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
                     Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));

SDValue Carry = Next.getValue(1);
Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
  return false;

if (UseGlue)
  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
                   Carry);
else
  Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
                   Zero, Carry);

Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

if (Opcode == ISD::SMUL_LOHI) {
  SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
                                DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
  Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);

  NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
                        DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
  Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
}

Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
return true;
6472}

6474bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
                             SelectionDAG &DAG, MulExpansionKind Kind,
                             SDValue LL, SDValue LH, SDValue RL,
                             SDValue RH) const {
SmallVector<SDValue, 2> Result;
bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
                         N->getOperand(0), N->getOperand(1), Result, HiLoVT,
                         DAG, Kind, LL, LH, RL, RH);
if (Ok) {
  assert(Result.size() == 2)(static_cast <bool> (Result.size() == 2) ? void (0) : __assert_fail
 ("Result.size() == 2", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6483, __extension__ __PRETTY_FUNCTION__));
  Lo = Result[0];
  Hi = Result[1];
}
return Ok;
6488}

6490// Check that (every element of) Z is undef or not an exact multiple of BW.
6491static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
return ISD::matchUnaryPredicate(
    Z,
    [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
    true);
6496}

6498bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
                                     SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);

if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
                      !isOperationLegalOrCustom(ISD::SRL, VT) ||
                      !isOperationLegalOrCustom(ISD::SUB, VT) ||
                      !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
  return false;

SDValue X = Node->getOperand(0);
SDValue Y = Node->getOperand(1);
SDValue Z = Node->getOperand(2);

unsigned BW = VT.getScalarSizeInBits();
bool IsFSHL = Node->getOpcode() == ISD::FSHL;
SDLoc DL(SDValue(Node, 0));

EVT ShVT = Z.getValueType();

// If a funnel shift in the other direction is more supported, use it.
unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
    isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
  if (isNonZeroModBitWidthOrUndef(Z, BW)) {
    // fshl X, Y, Z -> fshr X, Y, -Z
    // fshr X, Y, Z -> fshl X, Y, -Z
    SDValue Zero = DAG.getConstant(0, DL, ShVT);
    Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);
  } else {
    // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
    // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
    SDValue One = DAG.getConstant(1, DL, ShVT);
    if (IsFSHL) {
      Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
      X = DAG.getNode(ISD::SRL, DL, VT, X, One);
    } else {
      X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
      Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
    }
    Z = DAG.getNOT(DL, Z, ShVT);
  }
  Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
  return true;
}

SDValue ShX, ShY;
SDValue ShAmt, InvShAmt;
if (isNonZeroModBitWidthOrUndef(Z, BW)) {
  // fshl: X << C | Y >> (BW - C)
  // fshr: X << (BW - C) | Y >> C
  // where C = Z % BW is not zero
  SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
  ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
  InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
  ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
  ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
} else {
  // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
  // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
  SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
  if (isPowerOf2_32(BW)) {
    // Z % BW -> Z & (BW - 1)
    ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
    // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
    InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
  } else {
    SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
    ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
    InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
  }

  SDValue One = DAG.getConstant(1, DL, ShVT);
  if (IsFSHL) {
    ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
    SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
    ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
  } else {
    SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
    ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
    ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
  }
}
Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
return true;
6583}

6585// TODO: Merge with expandFunnelShift.
6586bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
                             SDValue &Result, SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
unsigned EltSizeInBits = VT.getScalarSizeInBits();
bool IsLeft = Node->getOpcode() == ISD::ROTL;
SDValue Op0 = Node->getOperand(0);
SDValue Op1 = Node->getOperand(1);
SDLoc DL(SDValue(Node, 0));

EVT ShVT = Op1.getValueType();
SDValue Zero = DAG.getConstant(0, DL, ShVT);

// If a rotate in the other direction is supported, use it.
unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
  SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
  Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
  return true;
}

if (!AllowVectorOps && VT.isVector() &&
    (!isOperationLegalOrCustom(ISD::SHL, VT) ||
     !isOperationLegalOrCustom(ISD::SRL, VT) ||
     !isOperationLegalOrCustom(ISD::SUB, VT) ||
     !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
     !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
  return false;

unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
SDValue ShVal;
SDValue HsVal;
if (isPowerOf2_32(EltSizeInBits)) {
  // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
  // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
  SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
  SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
  ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
  SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
  HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
} else {
  // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
  // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
  SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
  SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
  ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
  SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
  SDValue One = DAG.getConstant(1, DL, ShVT);
  HsVal =
      DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
}
Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
return true;
6640}

6642void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
                                    SelectionDAG &DAG) const {
assert(Node->getNumOperands() == 3 && "Not a double-shift!")(static_cast <bool> (Node->getNumOperands() == 3 &&
 "Not a double-shift!") ? void (0) : __assert_fail ("Node->getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6644, __extension__ __PRETTY_FUNCTION__));
EVT VT = Node->getValueType(0);
unsigned VTBits = VT.getScalarSizeInBits();
assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected")(static_cast <bool> (isPowerOf2_32(VTBits) && "Power-of-two integer type expected"
) ? void (0) : __assert_fail ("isPowerOf2_32(VTBits) && \"Power-of-two integer type expected\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6647, __extension__ __PRETTY_FUNCTION__));

bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
SDValue ShOpLo = Node->getOperand(0);
SDValue ShOpHi = Node->getOperand(1);
SDValue ShAmt = Node->getOperand(2);
EVT ShAmtVT = ShAmt.getValueType();
EVT ShAmtCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
SDLoc dl(Node);

// ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
// ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
// away during isel.
SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
                                DAG.getConstant(VTBits - 1, dl, ShAmtVT));
SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
                                   DAG.getConstant(VTBits - 1, dl, ShAmtVT))
                     : DAG.getConstant(0, dl, VT);

SDValue Tmp2, Tmp3;
if (IsSHL) {
  Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
} else {
  Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
  Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
}

// If the shift amount is larger or equal than the width of a part we don't
// use the result from the FSHL/FSHR. Insert a test and select the appropriate
// values for large shift amounts.
SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
                              DAG.getConstant(VTBits, dl, ShAmtVT));
SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
                            DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);

if (IsSHL) {
  Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
  Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
} else {
  Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
  Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
}
6692}

6694bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
                                    SelectionDAG &DAG) const {
unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
SDValue Src = Node->getOperand(OpNo);
EVT SrcVT = Src.getValueType();
EVT DstVT = Node->getValueType(0);
SDLoc dl(SDValue(Node, 0));

// FIXME: Only f32 to i64 conversions are supported.
if (SrcVT != MVT::f32 || DstVT != MVT::i64)
  return false;

if (Node->isStrictFPOpcode())
  // When a NaN is converted to an integer a trap is allowed. We can't
  // use this expansion here because it would eliminate that trap. Other
  // traps are also allowed and cannot be eliminated. See
  // IEEE 754-2008 sec 5.8.
  return false;

// Expand f32 -> i64 conversion
// This algorithm comes from compiler-rt's implementation of fixsfdi:
// https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
EVT IntVT = SrcVT.changeTypeToInteger();
EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());

SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
SDValue Bias = DAG.getConstant(127, dl, IntVT);
SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);

SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);

SDValue ExponentBits = DAG.getNode(
    ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
    DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);

SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
                           DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
                           DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);

SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
                        DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
                        DAG.getConstant(0x00800000, dl, IntVT));

R = DAG.getZExtOrTrunc(R, dl, DstVT);

R = DAG.getSelectCC(
    dl, Exponent, ExponentLoBit,
    DAG.getNode(ISD::SHL, dl, DstVT, R,
                DAG.getZExtOrTrunc(
                    DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
                    dl, IntShVT)),
    DAG.getNode(ISD::SRL, dl, DstVT, R,
                DAG.getZExtOrTrunc(
                    DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
                    dl, IntShVT)),
    ISD::SETGT);

SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
                          DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);

Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
                         DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
return true;
6763}

6765bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
                                    SDValue &Chain,
                                    SelectionDAG &DAG) const {
SDLoc dl(SDValue(Node, 0));
unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
SDValue Src = Node->getOperand(OpNo);

EVT SrcVT = Src.getValueType();
EVT DstVT = Node->getValueType(0);
EVT SetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
EVT DstSetCCVT =
    getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);

// Only expand vector types if we have the appropriate vector bit operations.
unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
                                                 ISD::FP_TO_SINT;
if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
                         !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))
  return false;

// If the maximum float value is smaller then the signed integer range,
// the destination signmask can't be represented by the float, so we can
// just use FP_TO_SINT directly.
const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
APFloat APF(APFSem, APInt::getZero(SrcVT.getScalarSizeInBits()));
APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
if (APFloat::opOverflow &
    APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
  if (Node->isStrictFPOpcode()) {
    Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
                         { Node->getOperand(0), Src });
    Chain = Result.getValue(1);
  } else
    Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
  return true;
}

// Don't expand it if there isn't cheap fsub instruction.
if (!isOperationLegalOrCustom(
        Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
  return false;

SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
SDValue Sel;

if (Node->isStrictFPOpcode()) {
  Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
                     Node->getOperand(0), /*IsSignaling*/ true);
  Chain = Sel.getValue(1);
} else {
  Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
}

bool Strict = Node->isStrictFPOpcode() ||
              shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);

if (Strict) {
  // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
  // signmask then offset (the result of which should be fully representable).
  // Sel = Src < 0x8000000000000000
  // FltOfs = select Sel, 0, 0x8000000000000000
  // IntOfs = select Sel, 0, 0x8000000000000000
  // Result = fp_to_sint(Src - FltOfs) ^ IntOfs

  // TODO: Should any fast-math-flags be set for the FSUB?
  SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
                                 DAG.getConstantFP(0.0, dl, SrcVT), Cst);
  Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
  SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
                                 DAG.getConstant(0, dl, DstVT),
                                 DAG.getConstant(SignMask, dl, DstVT));
  SDValue SInt;
  if (Node->isStrictFPOpcode()) {
    SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
                              { Chain, Src, FltOfs });
    SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
                       { Val.getValue(1), Val });
    Chain = SInt.getValue(1);
  } else {
    SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
    SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
  }
  Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
} else {
  // Expand based on maximum range of FP_TO_SINT:
  // True = fp_to_sint(Src)
  // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
  // Result = select (Src < 0x8000000000000000), True, False

  SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
  // TODO: Should any fast-math-flags be set for the FSUB?
  SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
                              DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
  False = DAG.getNode(ISD::XOR, dl, DstVT, False,
                      DAG.getConstant(SignMask, dl, DstVT));
  Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
  Result = DAG.getSelect(dl, DstVT, Sel, True, False);
}
return true;
6865}

6867bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
                                    SDValue &Chain,
                                    SelectionDAG &DAG) const {
// This transform is not correct for converting 0 when rounding mode is set
// to round toward negative infinity which will produce -0.0. So disable under
// strictfp.
if (Node->isStrictFPOpcode())
  return false;

SDValue Src = Node->getOperand(0);
EVT SrcVT = Src.getValueType();
EVT DstVT = Node->getValueType(0);

if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
  return false;

// Only expand vector types if we have the appropriate vector bit operations.
if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
                         !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
                         !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
                         !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||
                         !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))
  return false;

SDLoc dl(SDValue(Node, 0));
EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());

// Implementation of unsigned i64 to f64 following the algorithm in
// __floatundidf in compiler_rt.  This implementation performs rounding
// correctly in all rounding modes with the exception of converting 0
// when rounding toward negative infinity. In that case the fsub will produce
// -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.
SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000)0x4330000000000000UL, dl, SrcVT);
SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
    BitsToDouble(UINT64_C(0x4530000000100000)0x4530000000100000UL), dl, DstVT);
SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000)0x4530000000000000UL, dl, SrcVT);
SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF)0x00000000FFFFFFFFUL, dl, SrcVT);
SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);

SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
SDValue HiSub =
    DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
return true;
6916}

6918SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
                                            SelectionDAG &DAG) const {
SDLoc dl(Node);
unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
  ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
EVT VT = Node->getValueType(0);

if (VT.isScalableVector())
  report_fatal_error(
      "Expanding fminnum/fmaxnum for scalable vectors is undefined.");

if (isOperationLegalOrCustom(NewOp, VT)) {
  SDValue Quiet0 = Node->getOperand(0);
  SDValue Quiet1 = Node->getOperand(1);

  if (!Node->getFlags().hasNoNaNs()) {
    // Insert canonicalizes if it's possible we need to quiet to get correct
    // sNaN behavior.
    if (!DAG.isKnownNeverSNaN(Quiet0)) {
      Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
                           Node->getFlags());
    }
    if (!DAG.isKnownNeverSNaN(Quiet1)) {
      Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
                           Node->getFlags());
    }
  }

  return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
}

// If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
// instead if there are no NaNs.
if (Node->getFlags().hasNoNaNs()) {
  unsigned IEEE2018Op =
      Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
  if (isOperationLegalOrCustom(IEEE2018Op, VT)) {
    return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
                       Node->getOperand(1), Node->getFlags());
  }
}

// If none of the above worked, but there are no NaNs, then expand to
// a compare/select sequence.  This is required for correctness since
// InstCombine might have canonicalized a fcmp+select sequence to a
// FMINNUM/FMAXNUM node.  If we were to fall through to the default
// expansion to libcall, we might introduce a link-time dependency
// on libm into a file that originally did not have one.
if (Node->getFlags().hasNoNaNs()) {
  ISD::CondCode Pred =
      Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
  SDValue Op1 = Node->getOperand(0);
  SDValue Op2 = Node->getOperand(1);
  SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);
  // Copy FMF flags, but always set the no-signed-zeros flag
  // as this is implied by the FMINNUM/FMAXNUM semantics.
  SDNodeFlags Flags = Node->getFlags();
  Flags.setNoSignedZeros(true);
  SelCC->setFlags(Flags);
  return SelCC;
}

return SDValue();
6981}

6983// Only expand vector types if we have the appropriate vector bit operations.
6984static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
assert(VT.isVector() && "Expected vector type")(static_cast <bool> (VT.isVector() && "Expected vector type"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected vector type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 6985, __extension__ __PRETTY_FUNCTION__));
unsigned Len = VT.getScalarSizeInBits();
return TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
       TLI.isOperationLegalOrCustom(ISD::SUB, VT) &&
       TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
       (Len == 8 || TLI.isOperationLegalOrCustom(ISD::MUL, VT)) &&
       TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT);
6992}

6994SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
SDValue Op = Node->getOperand(0);
unsigned Len = VT.getScalarSizeInBits();
assert(VT.isInteger() && "CTPOP not implemented for this type.")(static_cast <bool> (VT.isInteger() && "CTPOP not implemented for this type."
) ? void (0) : __assert_fail ("VT.isInteger() && \"CTPOP not implemented for this type.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7000, __extension__ __PRETTY_FUNCTION__));

// TODO: Add support for irregular type lengths.
if (!(Len <= 128 && Len % 8 == 0))
  return SDValue();

// Only expand vector types if we have the appropriate vector bit operations.
if (VT.isVector() && !canExpandVectorCTPOP(*this, VT))
  return SDValue();

// This is the "best" algorithm from
// http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
SDValue Mask55 =
    DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
SDValue Mask33 =
    DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
SDValue Mask0F =
    DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
SDValue Mask01 =
    DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);

// v = v - ((v >> 1) & 0x55555555...)
Op = DAG.getNode(ISD::SUB, dl, VT, Op,
                 DAG.getNode(ISD::AND, dl, VT,
                             DAG.getNode(ISD::SRL, dl, VT, Op,
                                         DAG.getConstant(1, dl, ShVT)),
                             Mask55));
// v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
                 DAG.getNode(ISD::AND, dl, VT,
                             DAG.getNode(ISD::SRL, dl, VT, Op,
                                         DAG.getConstant(2, dl, ShVT)),
                             Mask33));
// v = (v + (v >> 4)) & 0x0F0F0F0F...
Op = DAG.getNode(ISD::AND, dl, VT,
                 DAG.getNode(ISD::ADD, dl, VT, Op,
                             DAG.getNode(ISD::SRL, dl, VT, Op,
                                         DAG.getConstant(4, dl, ShVT))),
                 Mask0F);
// v = (v * 0x01010101...) >> (Len - 8)
if (Len > 8)
  Op =
      DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
                  DAG.getConstant(Len - 8, dl, ShVT));

return Op;
7046}

7048SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
SDValue Op = Node->getOperand(0);
unsigned NumBitsPerElt = VT.getScalarSizeInBits();

// If the non-ZERO_UNDEF version is supported we can use that instead.
if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
    isOperationLegalOrCustom(ISD::CTLZ, VT))
  return DAG.getNode(ISD::CTLZ, dl, VT, Op);

// If the ZERO_UNDEF version is supported use that and handle the zero case.
if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
  SDValue Zero = DAG.getConstant(0, dl, VT);
  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
  return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
                     DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
}

// Only expand vector types if we have the appropriate vector bit operations.
// This includes the operations needed to expand CTPOP if it isn't supported.
if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
                      (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
                       !canExpandVectorCTPOP(*this, VT)) ||
                      !isOperationLegalOrCustom(ISD::SRL, VT) ||
                      !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
  return SDValue();

// for now, we do this:
// x = x | (x >> 1);
// x = x | (x >> 2);
// ...
// x = x | (x >>16);
// x = x | (x >>32); // for 64-bit input
// return popcount(~x);
//
// Ref: "Hacker's Delight" by Henry Warren
for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {
  SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
  Op = DAG.getNode(ISD::OR, dl, VT, Op,
                   DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
}
Op = DAG.getNOT(dl, Op, VT);
return DAG.getNode(ISD::CTPOP, dl, VT, Op);
7096}

7098SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
SDValue Op = Node->getOperand(0);
unsigned NumBitsPerElt = VT.getScalarSizeInBits();

// If the non-ZERO_UNDEF version is supported we can use that instead.
if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
    isOperationLegalOrCustom(ISD::CTTZ, VT))
  return DAG.getNode(ISD::CTTZ, dl, VT, Op);

// If the ZERO_UNDEF version is supported use that and handle the zero case.
if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
  EVT SetCCVT =
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
  SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
  SDValue Zero = DAG.getConstant(0, dl, VT);
  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
  return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
                     DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
}

// Only expand vector types if we have the appropriate vector bit operations.
// This includes the operations needed to expand CTPOP if it isn't supported.
if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
                      (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
                       !isOperationLegalOrCustom(ISD::CTLZ, VT) &&
                       !canExpandVectorCTPOP(*this, VT)) ||
                      !isOperationLegalOrCustom(ISD::SUB, VT) ||
                      !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
                      !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
  return SDValue();

// for now, we use: { return popcount(~x & (x - 1)); }
// unless the target has ctlz but not ctpop, in which case we use:
// { return 32 - nlz(~x & (x-1)); }
// Ref: "Hacker's Delight" by Henry Warren
SDValue Tmp = DAG.getNode(
    ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
    DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));

// If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
  return DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
                     DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
}

return DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
7146}

7148SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
                                bool IsNegative) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
SDValue Op = N->getOperand(0);

// abs(x) -> smax(x,sub(0,x))
if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
    isOperationLegal(ISD::SMAX, VT)) {
  SDValue Zero = DAG.getConstant(0, dl, VT);
  return DAG.getNode(ISD::SMAX, dl, VT, Op,
                     DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
}

// abs(x) -> umin(x,sub(0,x))
if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
    isOperationLegal(ISD::UMIN, VT)) {
  SDValue Zero = DAG.getConstant(0, dl, VT);
  return DAG.getNode(ISD::UMIN, dl, VT, Op,
                     DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
}

// 0 - abs(x) -> smin(x, sub(0,x))
if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
    isOperationLegal(ISD::SMIN, VT)) {
  SDValue Zero = DAG.getConstant(0, dl, VT);
  return DAG.getNode(ISD::SMIN, dl, VT, Op,
                     DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
}

// Only expand vector types if we have the appropriate vector operations.
if (VT.isVector() &&
    (!isOperationLegalOrCustom(ISD::SRA, VT) ||
     (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
     (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
     !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
  return SDValue();

SDValue Shift =
    DAG.getNode(ISD::SRA, dl, VT, Op,
                DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));
if (!IsNegative) {
  SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
  return DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
}

// 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
return DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
7198}

7200SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
SDValue Op = N->getOperand(0);

if (!VT.isSimple())
  return SDValue();

EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
switch (VT.getSimpleVT().getScalarType().SimpleTy) {
default:
  return SDValue();
case MVT::i16:
  // Use a rotate by 8. This can be further expanded if necessary.
  return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
case MVT::i32:
  Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
                     DAG.getConstant(0xFF0000, dl, VT));
  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
  return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
case MVT::i64:
  Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
  Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
  Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
  Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
  Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
  Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
  Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
                     DAG.getConstant(255ULL<<48, dl, VT));
  Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
                     DAG.getConstant(255ULL<<40, dl, VT));
  Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
                     DAG.getConstant(255ULL<<32, dl, VT));
  Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
                     DAG.getConstant(255ULL<<24, dl, VT));
  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
                     DAG.getConstant(255ULL<<16, dl, VT));
  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
                     DAG.getConstant(255ULL<<8 , dl, VT));
  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
  Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
  return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
}
7256}

7258SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
SDValue Op = N->getOperand(0);
EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
unsigned Sz = VT.getScalarSizeInBits();

SDValue Tmp, Tmp2, Tmp3;

// If we can, perform BSWAP first and then the mask+swap the i4, then i2
// and finally the i1 pairs.
// TODO: We can easily support i4/i2 legal types if any target ever does.
if (Sz >= 8 && isPowerOf2_32(Sz)) {
  // Create the masks - repeating the pattern every byte.
  APInt Mask4 = APInt::getSplat(Sz, APInt(8, 0x0F));
  APInt Mask2 = APInt::getSplat(Sz, APInt(8, 0x33));
  APInt Mask1 = APInt::getSplat(Sz, APInt(8, 0x55));

  // BSWAP if the type is wider than a single byte.
  Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);

  // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(4, dl, SHVT));
  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask4, dl, VT));
  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask4, dl, VT));
  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

  // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(2, dl, SHVT));
  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask2, dl, VT));
  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask2, dl, VT));
  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);

  // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp, DAG.getConstant(1, dl, SHVT));
  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Mask1, dl, VT));
  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(Mask1, dl, VT));
  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
  return Tmp;
}

Tmp = DAG.getConstant(0, dl, VT);
for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
  if (I < J)
    Tmp2 =
        DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
  else
    Tmp2 =
        DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));

  APInt Shift(Sz, 1);
  Shift <<= J;
  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
}

return Tmp;
7318}

7320std::pair<SDValue, SDValue>
7321TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
                                  SelectionDAG &DAG) const {
SDLoc SL(LD);
SDValue Chain = LD->getChain();
SDValue BasePTR = LD->getBasePtr();
EVT SrcVT = LD->getMemoryVT();
EVT DstVT = LD->getValueType(0);
ISD::LoadExtType ExtType = LD->getExtensionType();

if (SrcVT.isScalableVector())
  report_fatal_error("Cannot scalarize scalable vector loads");

unsigned NumElem = SrcVT.getVectorNumElements();

EVT SrcEltVT = SrcVT.getScalarType();
EVT DstEltVT = DstVT.getScalarType();

// A vector must always be stored in memory as-is, i.e. without any padding
// between the elements, since various code depend on it, e.g. in the
// handling of a bitcast of a vector type to int, which may be done with a
// vector store followed by an integer load. A vector that does not have
// elements that are byte-sized must therefore be stored as an integer
// built out of the extracted vector elements.
if (!SrcEltVT.isByteSized()) {
  unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
  EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);

  unsigned NumSrcBits = SrcVT.getSizeInBits();
  EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);

  unsigned SrcEltBits = SrcEltVT.getSizeInBits();
  SDValue SrcEltBitMask = DAG.getConstant(
      APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);

  // Load the whole vector and avoid masking off the top bits as it makes
  // the codegen worse.
  SDValue Load =
      DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
                     LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),
                     LD->getMemOperand()->getFlags(), LD->getAAInfo());

  SmallVector<SDValue, 8> Vals;
  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
    unsigned ShiftIntoIdx =
        (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
    SDValue ShiftAmount =
        DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),
                                   LoadVT, SL, /*LegalTypes=*/false);
    SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
    SDValue Elt =
        DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
    SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);

    if (ExtType != ISD::NON_EXTLOAD) {
      unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
      Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
    }

    Vals.push_back(Scalar);
  }

  SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
  return std::make_pair(Value, Load.getValue(1));
}

unsigned Stride = SrcEltVT.getSizeInBits() / 8;
assert(SrcEltVT.isByteSized())(static_cast <bool> (SrcEltVT.isByteSized()) ? void (0)
 : __assert_fail ("SrcEltVT.isByteSized()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7387, __extension__ __PRETTY_FUNCTION__));

SmallVector<SDValue, 8> Vals;
SmallVector<SDValue, 8> LoadChains;

for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
  SDValue ScalarLoad =
      DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
                     LD->getPointerInfo().getWithOffset(Idx * Stride),
                     SrcEltVT, LD->getOriginalAlign(),
                     LD->getMemOperand()->getFlags(), LD->getAAInfo());

  BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::Fixed(Stride));

  Vals.push_back(ScalarLoad.getValue(0));
  LoadChains.push_back(ScalarLoad.getValue(1));
}

SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);

return std::make_pair(Value, NewChain);
7409}

7411SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
                                           SelectionDAG &DAG) const {
SDLoc SL(ST);

SDValue Chain = ST->getChain();
SDValue BasePtr = ST->getBasePtr();
SDValue Value = ST->getValue();
EVT StVT = ST->getMemoryVT();

if (StVT.isScalableVector())
  report_fatal_error("Cannot scalarize scalable vector stores");

// The type of the data we want to save
EVT RegVT = Value.getValueType();
EVT RegSclVT = RegVT.getScalarType();

// The type of data as saved in memory.
EVT MemSclVT = StVT.getScalarType();

unsigned NumElem = StVT.getVectorNumElements();

// A vector must always be stored in memory as-is, i.e. without any padding
// between the elements, since various code depend on it, e.g. in the
// handling of a bitcast of a vector type to int, which may be done with a
// vector store followed by an integer load. A vector that does not have
// elements that are byte-sized must therefore be stored as an integer
// built out of the extracted vector elements.
if (!MemSclVT.isByteSized()) {
  unsigned NumBits = StVT.getSizeInBits();
  EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);

  SDValue CurrVal = DAG.getConstant(0, SL, IntVT);

  for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
    SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
                              DAG.getVectorIdxConstant(Idx, SL));
    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
    SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
    unsigned ShiftIntoIdx =
        (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
    SDValue ShiftAmount =
        DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
    SDValue ShiftedElt =
        DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
    CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
  }

  return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
                      ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
                      ST->getAAInfo());
}

// Store Stride in bytes
unsigned Stride = MemSclVT.getSizeInBits() / 8;
assert(Stride && "Zero stride!")(static_cast <bool> (Stride && "Zero stride!") ?
 void (0) : __assert_fail ("Stride && \"Zero stride!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7465, __extension__ __PRETTY_FUNCTION__));
// Extract each of the elements from the original vector and save them into
// memory individually.
SmallVector<SDValue, 8> Stores;
for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
                            DAG.getVectorIdxConstant(Idx, SL));

  SDValue Ptr =
      DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Idx * Stride));

  // This scalar TruncStore may be illegal, but we legalize it later.
  SDValue Store = DAG.getTruncStore(
      Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
      MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
      ST->getAAInfo());

  Stores.push_back(Store);
}

return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
7486}

7488std::pair<SDValue, SDValue>
7489TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
assert(LD->getAddressingMode() == ISD::UNINDEXED &&(static_cast <bool> (LD->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed loads not implemented!"
) ? void (0) : __assert_fail ("LD->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed loads not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7491, __extension__ __PRETTY_FUNCTION__))
       "unaligned indexed loads not implemented!")(static_cast <bool> (LD->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed loads not implemented!"
) ? void (0) : __assert_fail ("LD->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed loads not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7491, __extension__ __PRETTY_FUNCTION__));
SDValue Chain = LD->getChain();
SDValue Ptr = LD->getBasePtr();
EVT VT = LD->getValueType(0);
EVT LoadedVT = LD->getMemoryVT();
SDLoc dl(LD);
auto &MF = DAG.getMachineFunction();

if (VT.isFloatingPoint() || VT.isVector()) {
  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
  if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
    if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
        LoadedVT.isVector()) {
      // Scalarize the load and let the individual components be handled.
      return scalarizeVectorLoad(LD, DAG);
    }

    // Expand to a (misaligned) integer load of the same size,
    // then bitconvert to floating point or vector.
    SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
                                  LD->getMemOperand());
    SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
    if (LoadedVT != VT)
      Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
                           ISD::ANY_EXTEND, dl, VT, Result);

    return std::make_pair(Result, newLoad.getValue(1));
  }

  // Copy the value to a (aligned) stack slot using (unaligned) integer
  // loads and stores, then do a (aligned) load from the stack slot.
  MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
  unsigned LoadedBytes = LoadedVT.getStoreSize();
  unsigned RegBytes = RegVT.getSizeInBits() / 8;
  unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;

  // Make sure the stack slot is also aligned for the register type.
  SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
  auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
  SmallVector<SDValue, 8> Stores;
  SDValue StackPtr = StackBase;
  unsigned Offset = 0;

  EVT PtrVT = Ptr.getValueType();
  EVT StackPtrVT = StackPtr.getValueType();

  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);

  // Do all but one copies using the full register width.
  for (unsigned i = 1; i < NumRegs; i++) {
    // Load one integer register's worth from the original location.
    SDValue Load = DAG.getLoad(
        RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
        LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
        LD->getAAInfo());
    // Follow the load with a store to the stack slot.  Remember the store.
    Stores.push_back(DAG.getStore(
        Load.getValue(1), dl, Load, StackPtr,
        MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
    // Increment the pointers.
    Offset += RegBytes;

    Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
    StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
  }

  // The last copy may be partial.  Do an extending load.
  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
                                8 * (LoadedBytes - Offset));
  SDValue Load =
      DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
                     LD->getPointerInfo().getWithOffset(Offset), MemVT,
                     LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
                     LD->getAAInfo());
  // Follow the load with a store to the stack slot.  Remember the store.
  // On big-endian machines this requires a truncating store to ensure
  // that the bits end up in the right place.
  Stores.push_back(DAG.getTruncStore(
      Load.getValue(1), dl, Load, StackPtr,
      MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));

  // The order of the stores doesn't matter - say it with a TokenFactor.
  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);

  // Finally, perform the original load only redirected to the stack slot.
  Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
                        MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
                        LoadedVT);

  // Callers expect a MERGE_VALUES node.
  return std::make_pair(Load, TF);
}

assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&(static_cast <bool> (LoadedVT.isInteger() && !LoadedVT
.isVector() && "Unaligned load of unsupported type.")
 ? void (0) : __assert_fail ("LoadedVT.isInteger() && !LoadedVT.isVector() && \"Unaligned load of unsupported type.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7586, __extension__ __PRETTY_FUNCTION__))
       "Unaligned load of unsupported type.")(static_cast <bool> (LoadedVT.isInteger() && !LoadedVT
.isVector() && "Unaligned load of unsupported type.")
 ? void (0) : __assert_fail ("LoadedVT.isInteger() && !LoadedVT.isVector() && \"Unaligned load of unsupported type.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7586, __extension__ __PRETTY_FUNCTION__));

// Compute the new VT that is half the size of the old one.  This is an
// integer MVT.
unsigned NumBits = LoadedVT.getSizeInBits();
EVT NewLoadedVT;
NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
NumBits >>= 1;

Align Alignment = LD->getOriginalAlign();
unsigned IncrementSize = NumBits / 8;
ISD::LoadExtType HiExtType = LD->getExtensionType();

// If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
if (HiExtType == ISD::NON_EXTLOAD)
  HiExtType = ISD::ZEXTLOAD;

// Load the value in two parts
SDValue Lo, Hi;
if (DAG.getDataLayout().isLittleEndian()) {
  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
                      NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                      LD->getAAInfo());

  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
                      LD->getPointerInfo().getWithOffset(IncrementSize),
                      NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                      LD->getAAInfo());
} else {
  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
                      NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                      LD->getAAInfo());

  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
                      LD->getPointerInfo().getWithOffset(IncrementSize),
                      NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
                      LD->getAAInfo());
}

// aggregate the two parts
SDValue ShiftAmount =
    DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
                                                  DAG.getDataLayout()));
SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);

SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
                           Hi.getValue(1));

return std::make_pair(Result, TF);
7638}

7640SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
                                           SelectionDAG &DAG) const {
assert(ST->getAddressingMode() == ISD::UNINDEXED &&(static_cast <bool> (ST->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed stores not implemented!"
) ? void (0) : __assert_fail ("ST->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed stores not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7643, __extension__ __PRETTY_FUNCTION__))
       "unaligned indexed stores not implemented!")(static_cast <bool> (ST->getAddressingMode() == ISD::
UNINDEXED && "unaligned indexed stores not implemented!"
) ? void (0) : __assert_fail ("ST->getAddressingMode() == ISD::UNINDEXED && \"unaligned indexed stores not implemented!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7643, __extension__ __PRETTY_FUNCTION__));
SDValue Chain = ST->getChain();
SDValue Ptr = ST->getBasePtr();
SDValue Val = ST->getValue();
EVT VT = Val.getValueType();
Align Alignment = ST->getOriginalAlign();
auto &MF = DAG.getMachineFunction();
EVT StoreMemVT = ST->getMemoryVT();

SDLoc dl(ST);
if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
  if (isTypeLegal(intVT)) {
    if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
        StoreMemVT.isVector()) {
      // Scalarize the store and let the individual components be handled.
      SDValue Result = scalarizeVectorStore(ST, DAG);
      return Result;
    }
    // Expand to a bitconvert of the value to the integer type of the
    // same size, then a (misaligned) int store.
    // FIXME: Does not handle truncating floating point stores!
    SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
    Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
                          Alignment, ST->getMemOperand()->getFlags());
    return Result;
  }
  // Do a (aligned) store to a stack slot, then copy from the stack slot
  // to the final destination using (unaligned) integer loads and stores.
  MVT RegVT = getRegisterType(
      *DAG.getContext(),
      EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
  EVT PtrVT = Ptr.getValueType();
  unsigned StoredBytes = StoreMemVT.getStoreSize();
  unsigned RegBytes = RegVT.getSizeInBits() / 8;
  unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;

  // Make sure the stack slot is also aligned for the register type.
  SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();

  // Perform the original store, only redirected to the stack slot.
  SDValue Store = DAG.getTruncStore(
      Chain, dl, Val, StackPtr,
      MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);

  EVT StackPtrVT = StackPtr.getValueType();

  SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
  SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
  SmallVector<SDValue, 8> Stores;
  unsigned Offset = 0;

  // Do all but one copies using the full register width.
  for (unsigned i = 1; i < NumRegs; i++) {
    // Load one integer register's worth from the stack slot.
    SDValue Load = DAG.getLoad(
        RegVT, dl, Store, StackPtr,
        MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
    // Store it to the final location.  Remember the store.
    Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
                                  ST->getPointerInfo().getWithOffset(Offset),
                                  ST->getOriginalAlign(),
                                  ST->getMemOperand()->getFlags()));
    // Increment the pointers.
    Offset += RegBytes;
    StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
    Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
  }

  // The last store may be partial.  Do a truncating store.  On big-endian
  // machines this requires an extending load from the stack slot to ensure
  // that the bits are in the right place.
  EVT LoadMemVT =
      EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));

  // Load from the stack slot.
  SDValue Load = DAG.getExtLoad(
      ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
      MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);

  Stores.push_back(
      DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
                        ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
                        ST->getOriginalAlign(),
                        ST->getMemOperand()->getFlags(), ST->getAAInfo()));
  // The order of the stores doesn't matter - say it with a TokenFactor.
  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
  return Result;
}

assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&(static_cast <bool> (StoreMemVT.isInteger() && !
StoreMemVT.isVector() && "Unaligned store of unknown type."
) ? void (0) : __assert_fail ("StoreMemVT.isInteger() && !StoreMemVT.isVector() && \"Unaligned store of unknown type.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7735, __extension__ __PRETTY_FUNCTION__))
       "Unaligned store of unknown type.")(static_cast <bool> (StoreMemVT.isInteger() && !
StoreMemVT.isVector() && "Unaligned store of unknown type."
) ? void (0) : __assert_fail ("StoreMemVT.isInteger() && !StoreMemVT.isVector() && \"Unaligned store of unknown type.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7735, __extension__ __PRETTY_FUNCTION__));
// Get the half-size VT
EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
unsigned NumBits = NewStoredVT.getFixedSizeInBits();
unsigned IncrementSize = NumBits / 8;

// Divide the stored value in two parts.
SDValue ShiftAmount = DAG.getConstant(
    NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout()));
SDValue Lo = Val;
SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);

// Store the two parts
SDValue Store1, Store2;
Store1 = DAG.getTruncStore(Chain, dl,
                           DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
                           Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
                           ST->getMemOperand()->getFlags());

Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
Store2 = DAG.getTruncStore(
    Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
    ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
    ST->getMemOperand()->getFlags(), ST->getAAInfo());

SDValue Result =
    DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
return Result;
7763}

7765SDValue
7766TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
                                     const SDLoc &DL, EVT DataVT,
                                     SelectionDAG &DAG,
                                     bool IsCompressedMemory) const {
SDValue Increment;
EVT AddrVT = Addr.getValueType();
EVT MaskVT = Mask.getValueType();
assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&(static_cast <bool> (DataVT.getVectorElementCount() == MaskVT
.getVectorElementCount() && "Incompatible types of Data and Mask"
) ? void (0) : __assert_fail ("DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && \"Incompatible types of Data and Mask\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7774, __extension__ __PRETTY_FUNCTION__))
       "Incompatible types of Data and Mask")(static_cast <bool> (DataVT.getVectorElementCount() == MaskVT
.getVectorElementCount() && "Incompatible types of Data and Mask"
) ? void (0) : __assert_fail ("DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() && \"Incompatible types of Data and Mask\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7774, __extension__ __PRETTY_FUNCTION__));
if (IsCompressedMemory) {
  if (DataVT.isScalableVector())
    report_fatal_error(
        "Cannot currently handle compressed memory with scalable vectors");
  // Incrementing the pointer according to number of '1's in the mask.
  EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
  SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
  if (MaskIntVT.getSizeInBits() < 32) {
    MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
    MaskIntVT = MVT::i32;
  }

  // Count '1's with POPCNT.
  Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
  Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
  // Scale is an element size in bytes.
  SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
                                  AddrVT);
  Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
} else if (DataVT.isScalableVector()) {
  Increment = DAG.getVScale(DL, AddrVT,
                            APInt(AddrVT.getFixedSizeInBits(),
                                  DataVT.getStoreSize().getKnownMinSize()));
} else
  Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);

return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
7802}

7804static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
                                     EVT VecVT, const SDLoc &dl,
                                     ElementCount SubEC) {
assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&(static_cast <bool> (!(SubEC.isScalable() && VecVT
.isFixedLengthVector()) && "Cannot index a scalable vector within a fixed-width vector"
) ? void (0) : __assert_fail ("!(SubEC.isScalable() && VecVT.isFixedLengthVector()) && \"Cannot index a scalable vector within a fixed-width vector\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7808, __extension__ __PRETTY_FUNCTION__))
       "Cannot index a scalable vector within a fixed-width vector")(static_cast <bool> (!(SubEC.isScalable() && VecVT
.isFixedLengthVector()) && "Cannot index a scalable vector within a fixed-width vector"
) ? void (0) : __assert_fail ("!(SubEC.isScalable() && VecVT.isFixedLengthVector()) && \"Cannot index a scalable vector within a fixed-width vector\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7808, __extension__ __PRETTY_FUNCTION__));

unsigned NElts = VecVT.getVectorMinNumElements();
unsigned NumSubElts = SubEC.getKnownMinValue();
EVT IdxVT = Idx.getValueType();

if (VecVT.isScalableVector() && !SubEC.isScalable()) {
  // If this is a constant index and we know the value plus the number of the
  // elements in the subvector minus one is less than the minimum number of
  // elements then it's safe to return Idx.
  if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))
    if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
      return Idx;
  SDValue VS =
      DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));
  unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
  SDValue Sub = DAG.getNode(SubOpcode, dl, IdxVT, VS,
                            DAG.getConstant(NumSubElts, dl, IdxVT));
  return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);
}
if (isPowerOf2_32(NElts) && NumSubElts == 1) {
  APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));
  return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
                     DAG.getConstant(Imm, dl, IdxVT));
}
unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
                   DAG.getConstant(MaxIndex, dl, IdxVT));
7836}

7838SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
                                              SDValue VecPtr, EVT VecVT,
                                              SDValue Index) const {
return getVectorSubVecPointer(
    DAG, VecPtr, VecVT,
    EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(), 1),
    Index);
7845}

7847SDValue TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG,
                                             SDValue VecPtr, EVT VecVT,
                                             EVT SubVecVT,
                                             SDValue Index) const {
SDLoc dl(Index);
// Make sure the index type is big enough to compute in.
Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());

EVT EltVT = VecVT.getVectorElementType();

// Calculate the element offset and add it to the pointer.
unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&(static_cast <bool> (EltSize * 8 == EltVT.getFixedSizeInBits
() && "Converting bits to bytes lost precision") ? void
 (0) : __assert_fail ("EltSize * 8 == EltVT.getFixedSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7860, __extension__ __PRETTY_FUNCTION__))
       "Converting bits to bytes lost precision")(static_cast <bool> (EltSize * 8 == EltVT.getFixedSizeInBits
() && "Converting bits to bytes lost precision") ? void
 (0) : __assert_fail ("EltSize * 8 == EltVT.getFixedSizeInBits() && \"Converting bits to bytes lost precision\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7860, __extension__ __PRETTY_FUNCTION__));
assert(SubVecVT.getVectorElementType() == EltVT &&(static_cast <bool> (SubVecVT.getVectorElementType() ==
 EltVT && "Sub-vector must be a vector with matching element type"
) ? void (0) : __assert_fail ("SubVecVT.getVectorElementType() == EltVT && \"Sub-vector must be a vector with matching element type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7862, __extension__ __PRETTY_FUNCTION__))
       "Sub-vector must be a vector with matching element type")(static_cast <bool> (SubVecVT.getVectorElementType() ==
 EltVT && "Sub-vector must be a vector with matching element type"
) ? void (0) : __assert_fail ("SubVecVT.getVectorElementType() == EltVT && \"Sub-vector must be a vector with matching element type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7862, __extension__ __PRETTY_FUNCTION__));
Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl,
                                SubVecVT.getVectorElementCount());

EVT IdxVT = Index.getValueType();
if (SubVecVT.isScalableVector())
  Index =
      DAG.getNode(ISD::MUL, dl, IdxVT, Index,
                  DAG.getVScale(dl, IdxVT, APInt(IdxVT.getSizeInBits(), 1)));

Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
                    DAG.getConstant(EltSize, dl, IdxVT));
return DAG.getMemBasePlusOffset(VecPtr, Index, dl);
7875}

7877//===----------------------------------------------------------------------===//
7878// Implementation of Emulated TLS Model
7879//===----------------------------------------------------------------------===//

7881SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
                                              SelectionDAG &DAG) const {
// Access to address of TLS varialbe xyz is lowered to a function call:
//   __emutls_get_address( address of global variable named "__emutls_v.xyz" )
EVT PtrVT = getPointerTy(DAG.getDataLayout());
PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
SDLoc dl(GA);

ArgListTy Args;
ArgListEntry Entry;
std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
StringRef EmuTlsVarName(NameString);
GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
assert(EmuTlsVar && "Cannot find EmuTlsVar ")(static_cast <bool> (EmuTlsVar && "Cannot find EmuTlsVar "
) ? void (0) : __assert_fail ("EmuTlsVar && \"Cannot find EmuTlsVar \""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7895, __extension__ __PRETTY_FUNCTION__));
Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
Entry.Ty = VoidPtrType;
Args.push_back(Entry);

SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);

TargetLowering::CallLoweringInfo CLI(DAG);
CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);

// TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
// At last for X86 targets, maybe good for other targets too?
MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
MFI.setAdjustsStack(true); // Is this only for X86 target?
MFI.setHasCalls(true);

assert((GA->getOffset() == 0) &&(static_cast <bool> ((GA->getOffset() == 0) &&
 "Emulated TLS must have zero offset in GlobalAddressSDNode")
 ? void (0) : __assert_fail ("(GA->getOffset() == 0) && \"Emulated TLS must have zero offset in GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7914, __extension__ __PRETTY_FUNCTION__))
       "Emulated TLS must have zero offset in GlobalAddressSDNode")(static_cast <bool> ((GA->getOffset() == 0) &&
 "Emulated TLS must have zero offset in GlobalAddressSDNode")
 ? void (0) : __assert_fail ("(GA->getOffset() == 0) && \"Emulated TLS must have zero offset in GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7914, __extension__ __PRETTY_FUNCTION__));
return CallResult.first;
7916}

7918SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
                                              SelectionDAG &DAG) const {
assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.")(static_cast <bool> ((Op->getOpcode() == ISD::SETCC)
 && "Input has to be a SETCC node.") ? void (0) : __assert_fail
 ("(Op->getOpcode() == ISD::SETCC) && \"Input has to be a SETCC node.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7920, __extension__ __PRETTY_FUNCTION__));
if (!isCtlzFast())
  return SDValue();
ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
SDLoc dl(Op);
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
  if (C->isZero() && CC == ISD::SETEQ) {
    EVT VT = Op.getOperand(0).getValueType();
    SDValue Zext = Op.getOperand(0);
    if (VT.bitsLT(MVT::i32)) {
      VT = MVT::i32;
      Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
    }
    unsigned Log2b = Log2_32(VT.getSizeInBits());
    SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
    SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
                              DAG.getConstant(Log2b, dl, MVT::i32));
    return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
  }
}
return SDValue();
7941}

7943// Convert redundant addressing modes (e.g. scaling is redundant
7944// when accessing bytes).
7945ISD::MemIndexType
7946TargetLowering::getCanonicalIndexType(ISD::MemIndexType IndexType, EVT MemVT,
                                    SDValue Offsets) const {
bool IsScaledIndex =
    (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::UNSIGNED_SCALED);
bool IsSignedIndex =
    (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::SIGNED_UNSCALED);

// Scaling is unimportant for bytes, canonicalize to unscaled.
if (IsScaledIndex && MemVT.getScalarType() == MVT::i8)
  return IsSignedIndex ? ISD::SIGNED_UNSCALED : ISD::UNSIGNED_UNSCALED;

return IndexType;
7958}

7960SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
SDValue Op0 = Node->getOperand(0);
SDValue Op1 = Node->getOperand(1);
EVT VT = Op0.getValueType();
unsigned Opcode = Node->getOpcode();
SDLoc DL(Node);

// umin(x,y) -> sub(x,usubsat(x,y))
if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
    isOperationLegal(ISD::USUBSAT, VT)) {
  return DAG.getNode(ISD::SUB, DL, VT, Op0,
                     DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
}

// umax(x,y) -> add(x,usubsat(y,x))
if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
    isOperationLegal(ISD::USUBSAT, VT)) {
  return DAG.getNode(ISD::ADD, DL, VT, Op0,
                     DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
}

// Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
ISD::CondCode CC;
switch (Opcode) {
default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 7984);
case ISD::SMAX: CC = ISD::SETGT; break;
case ISD::SMIN: CC = ISD::SETLT; break;
case ISD::UMAX: CC = ISD::SETUGT; break;
case ISD::UMIN: CC = ISD::SETULT; break;
}

// FIXME: Should really try to split the vector in case it's legal on a
// subvector.
if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
  return DAG.UnrollVectorOp(Node);

SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
return DAG.getSelect(DL, VT, Cond, Op0, Op1);
7998}

8000SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
unsigned Opcode = Node->getOpcode();
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
EVT VT = LHS.getValueType();
SDLoc dl(Node);

assert(VT == RHS.getValueType() && "Expected operands to be the same type")(static_cast <bool> (VT == RHS.getValueType() &&
 "Expected operands to be the same type") ? void (0) : __assert_fail
 ("VT == RHS.getValueType() && \"Expected operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8007, __extension__ __PRETTY_FUNCTION__));
assert(VT.isInteger() && "Expected operands to be integers")(static_cast <bool> (VT.isInteger() && "Expected operands to be integers"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Expected operands to be integers\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8008, __extension__ __PRETTY_FUNCTION__));

// usub.sat(a, b) -> umax(a, b) - b
if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
  SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
  return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
}

// uadd.sat(a, b) -> umin(a, ~b) + b
if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
  SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
  SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
  return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
}

unsigned OverflowOp;
switch (Opcode) {
case ISD::SADDSAT:
  OverflowOp = ISD::SADDO;
  break;
case ISD::UADDSAT:
  OverflowOp = ISD::UADDO;
  break;
case ISD::SSUBSAT:
  OverflowOp = ISD::SSUBO;
  break;
case ISD::USUBSAT:
  OverflowOp = ISD::USUBO;
  break;
default:
  llvm_unreachable("Expected method to receive signed or unsigned saturation "::llvm::llvm_unreachable_internal("Expected method to receive signed or unsigned saturation "
 "addition or subtraction node.", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8039)
                   "addition or subtraction node.")::llvm::llvm_unreachable_internal("Expected method to receive signed or unsigned saturation "
 "addition or subtraction node.", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8039);
}

// FIXME: Should really try to split the vector in case it's legal on a
// subvector.
if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
  return DAG.UnrollVectorOp(Node);

unsigned BitWidth = LHS.getScalarValueSizeInBits();
EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
SDValue SumDiff = Result.getValue(0);
SDValue Overflow = Result.getValue(1);
SDValue Zero = DAG.getConstant(0, dl, VT);
SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);

if (Opcode == ISD::UADDSAT) {
  if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
    // (LHS + RHS) | OverflowMask
    SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
    return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
  }
  // Overflow ? 0xffff.... : (LHS + RHS)
  return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
}

if (Opcode == ISD::USUBSAT) {
  if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
    // (LHS - RHS) & ~OverflowMask
    SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
    SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
    return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
  }
  // Overflow ? 0 : (LHS - RHS)
  return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
}

// Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
APInt MinVal = APInt::getSignedMinValue(BitWidth);
SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
SDValue Shift = DAG.getNode(ISD::SRA, dl, VT, SumDiff,
                            DAG.getConstant(BitWidth - 1, dl, VT));
Result = DAG.getNode(ISD::XOR, dl, VT, Shift, SatMin);
return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
8083}

8085SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
unsigned Opcode = Node->getOpcode();
bool IsSigned = Opcode == ISD::SSHLSAT;
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
EVT VT = LHS.getValueType();
SDLoc dl(Node);

assert((Node->getOpcode() == ISD::SSHLSAT ||(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
 || Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8095, __extension__ __PRETTY_FUNCTION__))
        Node->getOpcode() == ISD::USHLSAT) &&(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
 || Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8095, __extension__ __PRETTY_FUNCTION__))
        "Expected a SHLSAT opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SSHLSAT
 || Node->getOpcode() == ISD::USHLSAT) && "Expected a SHLSAT opcode"
) ? void (0) : __assert_fail ("(Node->getOpcode() == ISD::SSHLSAT || Node->getOpcode() == ISD::USHLSAT) && \"Expected a SHLSAT opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8095, __extension__ __PRETTY_FUNCTION__));
assert(VT == RHS.getValueType() && "Expected operands to be the same type")(static_cast <bool> (VT == RHS.getValueType() &&
 "Expected operands to be the same type") ? void (0) : __assert_fail
 ("VT == RHS.getValueType() && \"Expected operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8096, __extension__ __PRETTY_FUNCTION__));
assert(VT.isInteger() && "Expected operands to be integers")(static_cast <bool> (VT.isInteger() && "Expected operands to be integers"
) ? void (0) : __assert_fail ("VT.isInteger() && \"Expected operands to be integers\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8097, __extension__ __PRETTY_FUNCTION__));

// If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.

unsigned BW = VT.getScalarSizeInBits();
SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
SDValue Orig =
    DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);

SDValue SatVal;
if (IsSigned) {
  SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
  SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
  SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT),
                           SatMin, SatMax, ISD::SETLT);
} else {
  SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
}
Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE);

return Result;
8118}

8120SDValue
8121TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
assert((Node->getOpcode() == ISD::SMULFIX ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
 || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
 void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8126, __extension__ __PRETTY_FUNCTION__))
        Node->getOpcode() == ISD::UMULFIX ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
 || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
 void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8126, __extension__ __PRETTY_FUNCTION__))
        Node->getOpcode() == ISD::SMULFIXSAT ||(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
 || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
 void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8126, __extension__ __PRETTY_FUNCTION__))
        Node->getOpcode() == ISD::UMULFIXSAT) &&(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
 || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
 void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8126, __extension__ __PRETTY_FUNCTION__))
       "Expected a fixed point multiplication opcode")(static_cast <bool> ((Node->getOpcode() == ISD::SMULFIX
 || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode
() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT
) && "Expected a fixed point multiplication opcode") ?
 void (0) : __assert_fail ("(Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::UMULFIX || Node->getOpcode() == ISD::SMULFIXSAT || Node->getOpcode() == ISD::UMULFIXSAT) && \"Expected a fixed point multiplication opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8126, __extension__ __PRETTY_FUNCTION__));

SDLoc dl(Node);
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
EVT VT = LHS.getValueType();
unsigned Scale = Node->getConstantOperandVal(2);
bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
                   Node->getOpcode() == ISD::UMULFIXSAT);
bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
               Node->getOpcode() == ISD::SMULFIXSAT);
EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
unsigned VTSize = VT.getScalarSizeInBits();

if (!Scale) {
  // [us]mul.fix(a, b, 0) -> mul(a, b)
  if (!Saturating) {
    if (isOperationLegalOrCustom(ISD::MUL, VT))
      return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
  } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
    SDValue Result =
        DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
    SDValue Product = Result.getValue(0);
    SDValue Overflow = Result.getValue(1);
    SDValue Zero = DAG.getConstant(0, dl, VT);

    APInt MinVal = APInt::getSignedMinValue(VTSize);
    APInt MaxVal = APInt::getSignedMaxValue(VTSize);
    SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
    SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
    // Xor the inputs, if resulting sign bit is 0 the product will be
    // positive, else negative.
    SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
    SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
    Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
    return DAG.getSelect(dl, VT, Overflow, Result, Product);
  } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
    SDValue Result =
        DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
    SDValue Product = Result.getValue(0);
    SDValue Overflow = Result.getValue(1);

    APInt MaxVal = APInt::getMaxValue(VTSize);
    SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
    return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
  }
}

assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
 "most the number of bits if unsigned.") ? void (0) : __assert_fail
 ("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8176, __extension__ __PRETTY_FUNCTION__))
       "Expected scale to be less than the number of bits if signed or at "(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
 "most the number of bits if unsigned.") ? void (0) : __assert_fail
 ("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8176, __extension__ __PRETTY_FUNCTION__))
       "most the number of bits if unsigned.")(static_cast <bool> (((Signed && Scale < VTSize
) || (!Signed && Scale <= VTSize)) && "Expected scale to be less than the number of bits if signed or at "
 "most the number of bits if unsigned.") ? void (0) : __assert_fail
 ("((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && \"Expected scale to be less than the number of bits if signed or at \" \"most the number of bits if unsigned.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8176, __extension__ __PRETTY_FUNCTION__));
assert(LHS.getValueType() == RHS.getValueType() &&(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Expected both operands to be the same type") ?
 void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Expected both operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8178, __extension__ __PRETTY_FUNCTION__))
       "Expected both operands to be the same type")(static_cast <bool> (LHS.getValueType() == RHS.getValueType
() && "Expected both operands to be the same type") ?
 void (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Expected both operands to be the same type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8178, __extension__ __PRETTY_FUNCTION__));

// Get the upper and lower bits of the result.
SDValue Lo, Hi;
unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
if (isOperationLegalOrCustom(LoHiOp, VT)) {
  SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
  Lo = Result.getValue(0);
  Hi = Result.getValue(1);
} else if (isOperationLegalOrCustom(HiOp, VT)) {
  Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
  Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
} else if (VT.isVector()) {
  return SDValue();
} else {
  report_fatal_error("Unable to expand fixed point multiplication.");
}

if (Scale == VTSize)
  // Result is just the top half since we'd be shifting by the width of the
  // operand. Overflow impossible so this works for both UMULFIX and
  // UMULFIXSAT.
  return Hi;

// The result will need to be shifted right by the scale since both operands
// are scaled. The result is given to us in 2 halves, so we only want part of
// both in the result.
EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
                             DAG.getConstant(Scale, dl, ShiftTy));
if (!Saturating)
  return Result;

if (!Signed) {
  // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
  // widened multiplication) aren't all zeroes.

  // Saturate to max if ((Hi >> Scale) != 0),
  // which is the same as if (Hi > ((1 << Scale) - 1))
  APInt MaxVal = APInt::getMaxValue(VTSize);
  SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
                                    dl, VT);
  Result = DAG.getSelectCC(dl, Hi, LowMask,
                           DAG.getConstant(MaxVal, dl, VT), Result,
                           ISD::SETUGT);

  return Result;
}

// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
// widened multiplication) aren't all ones or all zeroes.

SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);

if (Scale == 0) {
  SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
                             DAG.getConstant(VTSize - 1, dl, ShiftTy));
  SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
  // Saturated to SatMin if wide product is negative, and SatMax if wide
  // product is positive ...
  SDValue Zero = DAG.getConstant(0, dl, VT);
  SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
                                             ISD::SETLT);
  // ... but only if we overflowed.
  return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
}

//  We handled Scale==0 above so all the bits to examine is in Hi.

// Saturate to max if ((Hi >> (Scale - 1)) > 0),
// which is the same as if (Hi > (1 << (Scale - 1)) - 1)
SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
                                  dl, VT);
Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
// Saturate to min if (Hi >> (Scale - 1)) < -1),
// which is the same as if (HI < (-1 << (Scale - 1))
SDValue HighMask =
    DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
                    dl, VT);
Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
return Result;
8261}

8263SDValue
8264TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
                                  SDValue LHS, SDValue RHS,
                                  unsigned Scale, SelectionDAG &DAG) const {
assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
 ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8269, __extension__ __PRETTY_FUNCTION__))
        Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
 ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8269, __extension__ __PRETTY_FUNCTION__))
       "Expected a fixed point division opcode")(static_cast <bool> ((Opcode == ISD::SDIVFIX || Opcode ==
 ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::
UDIVFIXSAT) && "Expected a fixed point division opcode"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) && \"Expected a fixed point division opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8269, __extension__ __PRETTY_FUNCTION__));

EVT VT = LHS.getValueType();
bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

// If there is enough room in the type to upscale the LHS or downscale the
// RHS before the division, we can perform it in this type without having to
// resize. For signed operations, the LHS headroom is the number of
// redundant sign bits, and for unsigned ones it is the number of zeroes.
// The headroom for the RHS is the number of trailing zeroes.
unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
                          : DAG.computeKnownBits(LHS).countMinLeadingZeros();
unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();

// For signed saturating operations, we need to be able to detect true integer
// division overflow; that is, when you have MIN / -EPS. However, this
// is undefined behavior and if we emit divisions that could take such
// values it may cause undesired behavior (arithmetic exceptions on x86, for
// example).
// Avoid this by requiring an extra bit so that we never get this case.
// FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
// signed saturating division, we need to emit a whopping 32-bit division.
if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
  return SDValue();

unsigned LHSShift = std::min(LHSLead, Scale);
unsigned RHSShift = Scale - LHSShift;

// At this point, we know that if we shift the LHS up by LHSShift and the
// RHS down by RHSShift, we can emit a regular division with a final scaling
// factor of Scale.

EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
if (LHSShift)
  LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
                    DAG.getConstant(LHSShift, dl, ShiftTy));
if (RHSShift)
  RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
                    DAG.getConstant(RHSShift, dl, ShiftTy));

SDValue Quot;
if (Signed) {
  // For signed operations, if the resulting quotient is negative and the
  // remainder is nonzero, subtract 1 from the quotient to round towards
  // negative infinity.
  SDValue Rem;
  // FIXME: Ideally we would always produce an SDIVREM here, but if the
  // type isn't legal, SDIVREM cannot be expanded. There is no reason why
  // we couldn't just form a libcall, but the type legalizer doesn't do it.
  if (isTypeLegal(VT) &&
      isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
    Quot = DAG.getNode(ISD::SDIVREM, dl,
                       DAG.getVTList(VT, VT),
                       LHS, RHS);
    Rem = Quot.getValue(1);
    Quot = Quot.getValue(0);
  } else {
    Quot = DAG.getNode(ISD::SDIV, dl, VT,
                       LHS, RHS);
    Rem = DAG.getNode(ISD::SREM, dl, VT,
                      LHS, RHS);
  }
  SDValue Zero = DAG.getConstant(0, dl, VT);
  SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
  SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
  SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
  SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
  SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
                             DAG.getConstant(1, dl, VT));
  Quot = DAG.getSelect(dl, VT,
                       DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
                       Sub1, Quot);
} else
  Quot = DAG.getNode(ISD::UDIV, dl, VT,
                     LHS, RHS);

return Quot;
8348}

8350void TargetLowering::expandUADDSUBO(
  SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
SDLoc dl(Node);
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
bool IsAdd = Node->getOpcode() == ISD::UADDO;

// If ADD/SUBCARRY is legal, use that instead.
unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
  SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
  SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
                                  { LHS, RHS, CarryIn });
  Result = SDValue(NodeCarry.getNode(), 0);
  Overflow = SDValue(NodeCarry.getNode(), 1);
  return;
}

Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
                          LHS.getValueType(), LHS, RHS);

EVT ResultType = Node->getValueType(1);
EVT SetCCType = getSetCCResultType(
    DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8377}

8379void TargetLowering::expandSADDSUBO(
  SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
SDLoc dl(Node);
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
bool IsAdd = Node->getOpcode() == ISD::SADDO;

Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
                          LHS.getValueType(), LHS, RHS);

EVT ResultType = Node->getValueType(1);
EVT OType = getSetCCResultType(
    DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));

// If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
if (isOperationLegal(OpcSat, LHS.getValueType())) {
  SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
  SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
  Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
  return;
}

SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());

// For an addition, the result should be less than one of the operands (LHS)
// if and only if the other operand (RHS) is negative, otherwise there will
// be overflow.
// For a subtraction, the result should be less than one of the operands
// (LHS) if and only if the other operand (RHS) is (non-zero) positive,
// otherwise there will be overflow.
SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
SDValue ConditionRHS =
    DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);

Overflow = DAG.getBoolExtOrTrunc(
    DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
    ResultType, ResultType);
8417}

8419bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
                              SDValue &Overflow, SelectionDAG &DAG) const {
SDLoc dl(Node);
EVT VT = Node->getValueType(0);
EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
SDValue LHS = Node->getOperand(0);
SDValue RHS = Node->getOperand(1);
bool isSigned = Node->getOpcode() == ISD::SMULO;

// For power-of-two multiplications we can use a simpler shift expansion.
if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
  const APInt &C = RHSC->getAPIntValue();
  // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
  if (C.isPowerOf2()) {
    // smulo(x, signed_min) is same as umulo(x, signed_min).
    bool UseArithShift = isSigned && !C.isMinSignedValue();
    EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout());
    SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);
    Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
    Overflow = DAG.getSetCC(dl, SetCCVT,
        DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
                    dl, VT, Result, ShiftAmt),
        LHS, ISD::SETNE);
    return true;
  }
}

EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
if (VT.isVector())
  WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
                            VT.getVectorNumElements());

SDValue BottomHalf;
SDValue TopHalf;
static const unsigned Ops[2][3] =
    { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
      { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
  BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
  TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
} else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
  BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
                           RHS);
  TopHalf = BottomHalf.getValue(1);
} else if (isTypeLegal(WideVT)) {
  LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
  RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
  SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
  BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
  SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,
      getShiftAmountTy(WideVT, DAG.getDataLayout()));
  TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
                        DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
} else {
  if (VT.isVector())
    return false;

  // We can fall back to a libcall with an illegal type for the MUL if we
  // have a libcall big enough.
  // Also, we can fall back to a division in some cases, but that's a big
  // performance hit in the general case.
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
  if (WideVT == MVT::i16)
    LC = RTLIB::MUL_I16;
  else if (WideVT == MVT::i32)
    LC = RTLIB::MUL_I32;
  else if (WideVT == MVT::i64)
    LC = RTLIB::MUL_I64;
  else if (WideVT == MVT::i128)
    LC = RTLIB::MUL_I128;
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
 "Cannot expand this operation!") ? void (0) : __assert_fail (
"LC != RTLIB::UNKNOWN_LIBCALL && \"Cannot expand this operation!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8489, __extension__ __PRETTY_FUNCTION__));

  SDValue HiLHS;
  SDValue HiRHS;
  if (isSigned) {
    // The high part is obtained by SRA'ing all but one of the bits of low
    // part.
    unsigned LoSize = VT.getFixedSizeInBits();
    HiLHS =
        DAG.getNode(ISD::SRA, dl, VT, LHS,
                    DAG.getConstant(LoSize - 1, dl,
                                    getPointerTy(DAG.getDataLayout())));
    HiRHS =
        DAG.getNode(ISD::SRA, dl, VT, RHS,
                    DAG.getConstant(LoSize - 1, dl,
                                    getPointerTy(DAG.getDataLayout())));
  } else {
      HiLHS = DAG.getConstant(0, dl, VT);
      HiRHS = DAG.getConstant(0, dl, VT);
  }

  // Here we're passing the 2 arguments explicitly as 4 arguments that are
  // pre-lowered to the correct types. This all depends upon WideVT not
  // being a legal type for the architecture and thus has to be split to
  // two arguments.
  SDValue Ret;
  TargetLowering::MakeLibCallOptions CallOptions;
  CallOptions.setSExt(isSigned);
  CallOptions.setIsPostTypeLegalization(true);
  if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
    // Halves of WideVT are packed into registers in different order
    // depending on platform endianness. This is usually handled by
    // the C calling convention, but we can't defer to it in
    // the legalizer.
    SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
    Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
  } else {
    SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
    Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
  }
  assert(Ret.getOpcode() == ISD::MERGE_VALUES &&(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
 && "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8530, __extension__ __PRETTY_FUNCTION__))
         "Ret value is a collection of constituent nodes holding result.")(static_cast <bool> (Ret.getOpcode() == ISD::MERGE_VALUES
 && "Ret value is a collection of constituent nodes holding result."
) ? void (0) : __assert_fail ("Ret.getOpcode() == ISD::MERGE_VALUES && \"Ret value is a collection of constituent nodes holding result.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8530, __extension__ __PRETTY_FUNCTION__));
  if (DAG.getDataLayout().isLittleEndian()) {
    // Same as above.
    BottomHalf = Ret.getOperand(0);
    TopHalf = Ret.getOperand(1);
  } else {
    BottomHalf = Ret.getOperand(1);
    TopHalf = Ret.getOperand(0);
  }
}

Result = BottomHalf;
if (isSigned) {
  SDValue ShiftAmt = DAG.getConstant(
      VT.getScalarSizeInBits() - 1, dl,
      getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
  SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
  Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
} else {
  Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
                          DAG.getConstant(0, dl, VT), ISD::SETNE);
}

// Truncate the result if SetCC returns a larger type than needed.
EVT RType = Node->getValueType(1);
if (RType.bitsLT(Overflow.getValueType()))
  Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);

assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&(static_cast <bool> (RType.getSizeInBits() == Overflow.
getValueSizeInBits() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == Overflow.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8559, __extension__ __PRETTY_FUNCTION__))
       "Unexpected result type for S/UMULO legalization")(static_cast <bool> (RType.getSizeInBits() == Overflow.
getValueSizeInBits() && "Unexpected result type for S/UMULO legalization"
) ? void (0) : __assert_fail ("RType.getSizeInBits() == Overflow.getValueSizeInBits() && \"Unexpected result type for S/UMULO legalization\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8559, __extension__ __PRETTY_FUNCTION__));
return true;
8561}

8563SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
SDValue Op = Node->getOperand(0);
EVT VT = Op.getValueType();

if (VT.isScalableVector())
  report_fatal_error(
      "Expanding reductions for scalable vectors is undefined.");

// Try to use a shuffle reduction for power of two vectors.
if (VT.isPow2VectorType()) {
  while (VT.getVectorNumElements() > 1) {
    EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
    if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
      break;

    SDValue Lo, Hi;
    std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
    Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi);
    VT = HalfVT;
  }
}

EVT EltVT = VT.getVectorElementType();
unsigned NumElts = VT.getVectorNumElements();

SmallVector<SDValue, 8> Ops;
DAG.ExtractVectorElements(Op, Ops, 0, NumElts);

SDValue Res = Ops[0];
for (unsigned i = 1; i < NumElts; i++)
  Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());

// Result type may be wider than element type.
if (EltVT != Node->getValueType(0))
  Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
return Res;
8601}

8603SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
SDLoc dl(Node);
SDValue AccOp = Node->getOperand(0);
SDValue VecOp = Node->getOperand(1);
SDNodeFlags Flags = Node->getFlags();

EVT VT = VecOp.getValueType();
EVT EltVT = VT.getVectorElementType();

if (VT.isScalableVector())
  report_fatal_error(
      "Expanding reductions for scalable vectors is undefined.");

unsigned NumElts = VT.getVectorNumElements();

SmallVector<SDValue, 8> Ops;
DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);

unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());

SDValue Res = AccOp;
for (unsigned i = 0; i < NumElts; i++)
  Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);

return Res;
8628}

8630bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
                             SelectionDAG &DAG) const {
EVT VT = Node->getValueType(0);
SDLoc dl(Node);
bool isSigned = Node->getOpcode() == ISD::SREM;
unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
SDValue Dividend = Node->getOperand(0);
SDValue Divisor = Node->getOperand(1);
if (isOperationLegalOrCustom(DivRemOpc, VT)) {
  SDVTList VTs = DAG.getVTList(VT, VT);
  Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
  return true;
}
if (isOperationLegalOrCustom(DivOpc, VT)) {
  // X % Y -> X-X/Y*Y
  SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
  SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
  Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
  return true;
}
return false;
8652}

8654SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
                                          SelectionDAG &DAG) const {
bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
SDLoc dl(SDValue(Node, 0));
SDValue Src = Node->getOperand(0);

// DstVT is the result type, while SatVT is the size to which we saturate
EVT SrcVT = Src.getValueType();
EVT DstVT = Node->getValueType(0);

EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
unsigned SatWidth = SatVT.getScalarSizeInBits();
unsigned DstWidth = DstVT.getScalarSizeInBits();
assert(SatWidth <= DstWidth &&(static_cast <bool> (SatWidth <= DstWidth &&
 "Expected saturation width smaller than result width") ? void
 (0) : __assert_fail ("SatWidth <= DstWidth && \"Expected saturation width smaller than result width\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8668, __extension__ __PRETTY_FUNCTION__))
       "Expected saturation width smaller than result width")(static_cast <bool> (SatWidth <= DstWidth &&
 "Expected saturation width smaller than result width") ? void
 (0) : __assert_fail ("SatWidth <= DstWidth && \"Expected saturation width smaller than result width\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8668, __extension__ __PRETTY_FUNCTION__));

// Determine minimum and maximum integer values and their corresponding
// floating-point values.
APInt MinInt, MaxInt;
if (IsSigned) {
  MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth);
  MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth);
} else {
  MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth);
  MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth);
}

// We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as
// libcall emission cannot handle this. Large result types will fail.
if (SrcVT == MVT::f16) {
  Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
  SrcVT = Src.getValueType();
}

APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));

APFloat::opStatus MinStatus =
    MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
APFloat::opStatus MaxStatus =
    MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
                           !(MaxStatus & APFloat::opStatus::opInexact);

SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);

// If the integer bounds are exactly representable as floats and min/max are
// legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
// of comparisons and selects.
bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&
                   isOperationLegal(ISD::FMAXNUM, SrcVT);
if (AreExactFloatBounds && MinMaxLegal) {
  SDValue Clamped = Src;

  // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
  Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);
  // Clamp by MaxFloat from above. NaN cannot occur.
  Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);
  // Convert clamped value to integer.
  SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
                                dl, DstVT, Clamped);

  // In the unsigned case we're done, because we mapped NaN to MinFloat,
  // which will cast to zero.
  if (!IsSigned)
    return FpToInt;

  // Otherwise, select 0 if Src is NaN.
  SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
  return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
                         ISD::CondCode::SETUO);
}

SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);

// Result of direct conversion. The assumption here is that the operation is
// non-trapping and it's fine to apply it to an out-of-range value if we
// select it away later.
SDValue FpToInt =
    DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);

SDValue Select = FpToInt;

// If Src ULT MinFloat, select MinInt. In particular, this also selects
// MinInt if Src is NaN.
Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,
                         ISD::CondCode::SETULT);
// If Src OGT MaxFloat, select MaxInt.
Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
                         ISD::CondCode::SETOGT);

// In the unsigned case we are done, because we mapped NaN to MinInt, which
// is already zero.
if (!IsSigned)
  return Select;

// Otherwise, select 0 if Src is NaN.
SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
8755}

8757SDValue TargetLowering::expandVectorSplice(SDNode *Node,
                                         SelectionDAG &DAG) const {
assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!")(static_cast <bool> (Node->getOpcode() == ISD::VECTOR_SPLICE
 && "Unexpected opcode!") ? void (0) : __assert_fail (
"Node->getOpcode() == ISD::VECTOR_SPLICE && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8759, __extension__ __PRETTY_FUNCTION__));
assert(Node->getValueType(0).isScalableVector() &&(static_cast <bool> (Node->getValueType(0).isScalableVector
() && "Fixed length vector types expected to use SHUFFLE_VECTOR!"
) ? void (0) : __assert_fail ("Node->getValueType(0).isScalableVector() && \"Fixed length vector types expected to use SHUFFLE_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8761, __extension__ __PRETTY_FUNCTION__))
       "Fixed length vector types expected to use SHUFFLE_VECTOR!")(static_cast <bool> (Node->getValueType(0).isScalableVector
() && "Fixed length vector types expected to use SHUFFLE_VECTOR!"
) ? void (0) : __assert_fail ("Node->getValueType(0).isScalableVector() && \"Fixed length vector types expected to use SHUFFLE_VECTOR!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8761, __extension__ __PRETTY_FUNCTION__));

EVT VT = Node->getValueType(0);
SDValue V1 = Node->getOperand(0);
SDValue V2 = Node->getOperand(1);
int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();
SDLoc DL(Node);

// Expand through memory thusly:
//  Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
//  Store V1, Ptr
//  Store V2, Ptr + sizeof(V1)
//  If (Imm < 0)
//    TrailingElts = -Imm
//    Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))
//  else
//    Ptr = Ptr + (Imm * sizeof(VT.Elt))
//  Res = Load Ptr

Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);

EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
                             VT.getVectorElementCount() * 2);
SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
EVT PtrVT = StackPtr.getValueType();
auto &MF = DAG.getMachineFunction();
auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);

// Store the lo part of CONCAT_VECTORS(V1, V2)
SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);
// Store the hi part of CONCAT_VECTORS(V1, V2)
SDValue OffsetToV2 = DAG.getVScale(
    DL, PtrVT,
    APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);
SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);

if (Imm >= 0) {
  // Load back the required element. getVectorElementPointer takes care of
  // clamping the index if it's out-of-bounds.
  StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));
  // Load the spliced result
  return DAG.getLoad(VT, DL, StoreV2, StackPtr,
                     MachinePointerInfo::getUnknownStack(MF));
}

uint64_t TrailingElts = -Imm;

// NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.
TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();
SDValue TrailingBytes =
    DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);

if (TrailingElts > VT.getVectorMinNumElements()) {
  SDValue VLBytes = DAG.getVScale(
      DL, PtrVT,
      APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
  TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);
}

// Calculate the start address of the spliced result.
StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);

// Load the spliced result
return DAG.getLoad(VT, DL, StoreV2, StackPtr2,
                   MachinePointerInfo::getUnknownStack(MF));
8828}

8830bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
                                         SDValue &LHS, SDValue &RHS,
                                         SDValue &CC, bool &NeedInvert,
                                         const SDLoc &dl, SDValue &Chain,
                                         bool IsSignaling) const {
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
MVT OpVT = LHS.getSimpleValueType();
ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
NeedInvert = false;
switch (TLI.getCondCodeAction(CCCode, OpVT)) {
default:
  llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8841);
case TargetLowering::Legal:
  // Nothing to do.
  break;
case TargetLowering::Expand: {
  ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
  if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
    std::swap(LHS, RHS);
    CC = DAG.getCondCode(InvCC);
    return true;
  }
  // Swapping operands didn't work. Try inverting the condition.
  bool NeedSwap = false;
  InvCC = getSetCCInverse(CCCode, OpVT);
  if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
    // If inverting the condition is not enough, try swapping operands
    // on top of it.
    InvCC = ISD::getSetCCSwappedOperands(InvCC);
    NeedSwap = true;
  }
  if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
    CC = DAG.getCondCode(InvCC);
    NeedInvert = true;
    if (NeedSwap)
      std::swap(LHS, RHS);
    return true;
  }

  ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
  unsigned Opc = 0;
  switch (CCCode) {
  default:
    llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8873);
  case ISD::SETUO:
    if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
      CC1 = ISD::SETUNE;
      CC2 = ISD::SETUNE;
      Opc = ISD::OR;
      break;
    }
    assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETUE is expanded, SETOEQ or SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETUE is expanded, SETOEQ or SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8882, __extension__ __PRETTY_FUNCTION__))
           "If SETUE is expanded, SETOEQ or SETUNE must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETUE is expanded, SETOEQ or SETUNE must be legal!"
) ? void (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETUE is expanded, SETOEQ or SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8882, __extension__ __PRETTY_FUNCTION__));
    NeedInvert = true;
    LLVM_FALLTHROUGH[[gnu::fallthrough]];
  case ISD::SETO:
    assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
 (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8887, __extension__ __PRETTY_FUNCTION__))
           "If SETO is expanded, SETOEQ must be legal!")(static_cast <bool> (TLI.isCondCodeLegal(ISD::SETOEQ, OpVT
) && "If SETO is expanded, SETOEQ must be legal!") ? void
 (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8887, __extension__ __PRETTY_FUNCTION__));
    CC1 = ISD::SETOEQ;
    CC2 = ISD::SETOEQ;
    Opc = ISD::AND;
    break;
  case ISD::SETONE:
  case ISD::SETUEQ:
    // If the SETUO or SETO CC isn't legal, we might be able to use
    // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
    // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
    // the operands.
    CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
    if (!TLI.isCondCodeLegal(CC2, OpVT) &&
        (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
         TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
      CC1 = ISD::SETOGT;
      CC2 = ISD::SETOLT;
      Opc = ISD::OR;
      NeedInvert = ((unsigned)CCCode & 0x8U);
      break;
    }
    LLVM_FALLTHROUGH[[gnu::fallthrough]];
  case ISD::SETOEQ:
  case ISD::SETOGT:
  case ISD::SETOGE:
  case ISD::SETOLT:
  case ISD::SETOLE:
  case ISD::SETUNE:
  case ISD::SETUGT:
  case ISD::SETUGE:
  case ISD::SETULT:
  case ISD::SETULE:
    // If we are floating point, assign and break, otherwise fall through.
    if (!OpVT.isInteger()) {
      // We can use the 4th bit to tell if we are the unordered
      // or ordered version of the opcode.
      CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
      Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
      CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
      break;
    }
    // Fallthrough if we are unsigned integer.
    LLVM_FALLTHROUGH[[gnu::fallthrough]];
  case ISD::SETLE:
  case ISD::SETGT:
  case ISD::SETGE:
  case ISD::SETLT:
  case ISD::SETNE:
  case ISD::SETEQ:
    // If all combinations of inverting the condition and swapping operands
    // didn't work then we have no means to expand the condition.
    llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 8938);
  }

  SDValue SetCC1, SetCC2;
  if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
    // If we aren't the ordered or unorder operation,
    // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
    SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
    SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
  } else {
    // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
    SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
    SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
  }
  if (Chain)
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
                        SetCC2.getValue(1));
  LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
  RHS = SDValue();
  CC = SDValue();
  return true;
}
}
return false;
8962}

←

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

→

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG.  These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//

18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H

21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>

57namespace llvm {

59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;

72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                  bool force = false);

75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG.  Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
const EVT *VTs;
unsigned int NumVTs;
82};

84namespace ISD {

/// Node predicates

88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);

93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
                                bool BuildVectorOnly = false);

99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
                                 bool BuildVectorOnly = false);

105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);

109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);

113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);

117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);

121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);

125} // end namespace ISD

127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation.  Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node.  This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
friend struct DenseMapInfo<SDValue>;

SDNode *Node = nullptr; // The node defining the value we are using.
unsigned ResNo = 0;     // Which return value of the node we are using.

144public:
SDValue() = default;
SDValue(SDNode *node, unsigned resno);

/// get the index which selects a specific result in the SDNode
unsigned getResNo() const { return ResNo; }

/// get the SDNode which holds the desired result
SDNode *getNode() const { return Node; }

/// set the SDNode
void setNode(SDNode *N) { Node = N; }

inline SDNode *operator->() const { return Node; }

bool operator==(const SDValue &O) const {
  return Node == O.Node && ResNo == O.ResNo;
}
bool operator!=(const SDValue &O) const {
  return !operator==(O);
}
bool operator<(const SDValue &O) const {
  return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
}
explicit operator bool() const {
  return Node != nullptr;
}

SDValue getValue(unsigned R) const {
  return SDValue(Node, R);
}

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return the ValueType of the referenced return value.
inline EVT getValueType() const;

/// Return the simple ValueType of the referenced return value.
MVT getSimpleValueType() const {
  return getValueType().getSimpleVT();
}

/// Returns the size of the value in bits.
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits() const {
  return getValueType().getSizeInBits();
}

uint64_t getScalarValueSizeInBits() const {
  return getValueType().getScalarType().getFixedSizeInBits();
}

// Forwarding methods - These forward to the corresponding methods in SDNode.
inline unsigned getOpcode() const;
inline unsigned getNumOperands() const;
inline const SDValue &getOperand(unsigned i) const;
inline uint64_t getConstantOperandVal(unsigned i) const;
inline const APInt &getConstantOperandAPInt(unsigned i) const;
inline bool isTargetMemoryOpcode() const;
inline bool isTargetOpcode() const;
inline bool isMachineOpcode() const;
inline bool isUndef() const;
inline unsigned getMachineOpcode() const;
inline const DebugLoc &getDebugLoc() const;
inline void dump() const;
inline void dump(const SelectionDAG *G) const;
inline void dumpr() const;
inline void dumpr(const SelectionDAG *G) const;

/// Return true if this operand (which must be a chain) reaches the
/// specified operand without crossing any side-effecting instructions.
/// In practice, this looks through token factors and non-volatile loads.
/// In order to remain efficient, this only
/// looks a couple of nodes in, it does not do an exhaustive search.
bool reachesChainWithoutSideEffects(SDValue Dest,
                                    unsigned Depth = 2) const;

/// Return true if there are no nodes using value ResNo of Node.
inline bool use_empty() const;

/// Return true if there is exactly one node using value ResNo of Node.
inline bool hasOneUse() const;
230};

232template<> struct DenseMapInfo<SDValue> {
static inline SDValue getEmptyKey() {
  SDValue V;
  V.ResNo = -1U;
  return V;
}

static inline SDValue getTombstoneKey() {
  SDValue V;
  V.ResNo = -2U;
  return V;
}

static unsigned getHashValue(const SDValue &Val) {
  return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
          (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
}

static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
  return LHS == RHS;
}
253};

255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDValue &Val) {
  return Val.getNode();
}
263};
264template<> struct simplify_type<const SDValue> {
using SimpleType = /*const*/ SDNode *;

static SimpleType getSimplifiedValue(const SDValue &Val) {
  return Val.getNode();
}
270};

272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
/// Val - The value being used.
SDValue Val;
/// User - The user of this value.
SDNode *User = nullptr;
/// Prev, Next - Pointers to the uses list of the SDNode referred by
/// this operand.
SDUse **Prev = nullptr;
SDUse *Next = nullptr;

287public:
SDUse() = default;
SDUse(const SDUse &U) = delete;
SDUse &operator=(const SDUse &) = delete;

/// Normally SDUse will just implicitly convert to an SDValue that it holds.
operator const SDValue&() const { return Val; }

/// If implicit conversion to SDValue doesn't work, the get() method returns
/// the SDValue.
const SDValue &get() const { return Val; }

/// This returns the SDNode that contains this Use.
SDNode *getUser() { return User; }

/// Get the next SDUse in the use list.
SDUse *getNext() const { return Next; }

/// Convenience function for get().getNode().
SDNode *getNode() const { return Val.getNode(); }
/// Convenience function for get().getResNo().
unsigned getResNo() const { return Val.getResNo(); }
/// Convenience function for get().getValueType().
EVT getValueType() const { return Val.getValueType(); }

/// Convenience function for get().operator==
bool operator==(const SDValue &V) const {
  return Val == V;
}

/// Convenience function for get().operator!=
bool operator!=(const SDValue &V) const {
  return Val != V;
}

/// Convenience function for get().operator<
bool operator<(const SDValue &V) const {
  return Val < V;
}

327private:
friend class SelectionDAG;
friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

void setUser(SDNode *p) { User = p; }

/// Remove this use from its existing use list, assign it the
/// given value, and add it to the new value's node's use list.
inline void set(const SDValue &V);
/// Like set, but only supports initializing a newly-allocated
/// SDUse with a non-null value.
inline void setInitial(const SDValue &V);
/// Like set, but only sets the Node portion of the value,
/// leaving the ResNo portion unmodified.
inline void setNode(SDNode *N);

void addToList(SDUse **List) {
  Next = *List;
  if (Next) Next->Prev = &Next;
  Prev = List;
  *List = this;
}

void removeFromList() {
  *Prev = Next;
  if (Next) Next->Prev = Prev;
}
356};

358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
using SimpleType = SDNode *;

static SimpleType getSimplifiedValue(SDUse &Val) {
  return Val.getNode();
}
366};

368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
bool NoUnsignedWrap : 1;
bool NoSignedWrap : 1;
bool Exact : 1;
bool NoNaNs : 1;
bool NoInfs : 1;
bool NoSignedZeros : 1;
bool AllowReciprocal : 1;
bool AllowContract : 1;
bool ApproximateFuncs : 1;
bool AllowReassociation : 1;

// We assume instructions do not raise floating-point exceptions by default,
// and only those marked explicitly may do so.  We could choose to represent
// this via a positive "FPExcept" flags like on the MI level, but having a
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
// intersection logic more straightforward.
bool NoFPExcept : 1;

391public:
/// Default constructor turns off all optimization flags.
SDNodeFlags()
    : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
      NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
      AllowContract(false), ApproximateFuncs(false),
      AllowReassociation(false), NoFPExcept(false) {}

/// Propagate the fast-math-flags from an IR FPMathOperator.
void copyFMF(const FPMathOperator &FPMO) {
  setNoNaNs(FPMO.hasNoNaNs());
  setNoInfs(FPMO.hasNoInfs());
  setNoSignedZeros(FPMO.hasNoSignedZeros());
  setAllowReciprocal(FPMO.hasAllowReciprocal());
  setAllowContract(FPMO.hasAllowContract());
  setApproximateFuncs(FPMO.hasApproxFunc());
  setAllowReassociation(FPMO.hasAllowReassoc());
}

// These are mutators for each flag.
void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
void setNoSignedWrap(bool b) { NoSignedWrap = b; }
void setExact(bool b) { Exact = b; }
void setNoNaNs(bool b) { NoNaNs = b; }
void setNoInfs(bool b) { NoInfs = b; }
void setNoSignedZeros(bool b) { NoSignedZeros = b; }
void setAllowReciprocal(bool b) { AllowReciprocal = b; }
void setAllowContract(bool b) { AllowContract = b; }
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
void setAllowReassociation(bool b) { AllowReassociation = b; }
void setNoFPExcept(bool b) { NoFPExcept = b; }

// These are accessors for each flag.
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
bool hasNoSignedWrap() const { return NoSignedWrap; }
bool hasExact() const { return Exact; }
bool hasNoNaNs() const { return NoNaNs; }
bool hasNoInfs() const { return NoInfs; }
bool hasNoSignedZeros() const { return NoSignedZeros; }
bool hasAllowReciprocal() const { return AllowReciprocal; }
bool hasAllowContract() const { return AllowContract; }
bool hasApproximateFuncs() const { return ApproximateFuncs; }
bool hasAllowReassociation() const { return AllowReassociation; }
bool hasNoFPExcept() const { return NoFPExcept; }

/// Clear any flags in this flag set that aren't also set in Flags. All
/// flags will be cleared if Flags are undefined.
void intersectWith(const SDNodeFlags Flags) {
  NoUnsignedWrap &= Flags.NoUnsignedWrap;
  NoSignedWrap &= Flags.NoSignedWrap;
  Exact &= Flags.Exact;
  NoNaNs &= Flags.NoNaNs;
  NoInfs &= Flags.NoInfs;
  NoSignedZeros &= Flags.NoSignedZeros;
  AllowReciprocal &= Flags.AllowReciprocal;
  AllowContract &= Flags.AllowContract;
  ApproximateFuncs &= Flags.ApproximateFuncs;
  AllowReassociation &= Flags.AllowReassociation;
  NoFPExcept &= Flags.NoFPExcept;
}
451};

453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
/// The operation that this node performs.
int16_t NodeType;

460protected:
// We define a set of mini-helper classes to help us interpret the bits in our
// SubclassData.  These are designed to fit within a uint16_t so they pack
// with NodeType.

465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif

475BEGIN_TWO_BYTE_PACK()
class SDNodeBitfields {
  friend class SDNode;
  friend class MemIntrinsicSDNode;
  friend class MemSDNode;
  friend class SelectionDAG;

  uint16_t HasDebugValue : 1;
  uint16_t IsMemIntrinsic : 1;
  uint16_t IsDivergent : 1;
};
enum { NumSDNodeBits = 3 };

class ConstantSDNodeBitfields {
  friend class ConstantSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsOpaque : 1;
};

class MemSDNodeBitfields {
  friend class MemSDNode;
  friend class MemIntrinsicSDNode;
  friend class AtomicSDNode;

  uint16_t : NumSDNodeBits;

  uint16_t IsVolatile : 1;
  uint16_t IsNonTemporal : 1;
  uint16_t IsDereferenceable : 1;
  uint16_t IsInvariant : 1;
};
enum { NumMemSDNodeBits = NumSDNodeBits + 4 };

class LSBaseSDNodeBitfields {
  friend class LSBaseSDNode;
  friend class VPLoadStoreSDNode;
  friend class MaskedLoadStoreSDNode;
  friend class MaskedGatherScatterSDNode;
  friend class VPGatherScatterSDNode;

  uint16_t : NumMemSDNodeBits;

  // This storage is shared between disparate class hierarchies to hold an
  // enumeration specific to the class hierarchy in use.
  //   LSBaseSDNode => enum ISD::MemIndexedMode
  //   VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
  //   VPGatherScatterSDNode => enum ISD::MemIndexType
  //   MaskedGatherScatterSDNode => enum ISD::MemIndexType
  uint16_t AddressingMode : 3;
};
enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };

class LoadSDNodeBitfields {
  friend class LoadSDNode;
  friend class VPLoadSDNode;
  friend class MaskedLoadSDNode;
  friend class MaskedGatherSDNode;
  friend class VPGatherSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t ExtTy : 2; // enum ISD::LoadExtType
  uint16_t IsExpanding : 1;
};

class StoreSDNodeBitfields {
  friend class StoreSDNode;
  friend class VPStoreSDNode;
  friend class MaskedStoreSDNode;
  friend class MaskedScatterSDNode;
  friend class VPScatterSDNode;

  uint16_t : NumLSBaseSDNodeBits;

  uint16_t IsTruncating : 1;
  uint16_t IsCompressing : 1;
};

union {
  char RawSDNodeBits[sizeof(uint16_t)];
  SDNodeBitfields SDNodeBits;
  ConstantSDNodeBitfields ConstantSDNodeBits;
  MemSDNodeBitfields MemSDNodeBits;
  LSBaseSDNodeBitfields LSBaseSDNodeBits;
  LoadSDNodeBitfields LoadSDNodeBits;
  StoreSDNodeBitfields StoreSDNodeBits;
};
565END_TWO_BYTE_PACK()
566#undef BEGIN_TWO_BYTE_PACK
567#undef END_TWO_BYTE_PACK

// RawSDNodeBits must cover the entirety of the union.  This means that all of
// the union's members must have size <= RawSDNodeBits.  We write the RHS as
// "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");

579private:
friend class SelectionDAG;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;

/// Unique id per SDNode in the DAG.
int NodeId = -1;

/// The values that are used by this operation.
SDUse *OperandList = nullptr;

/// The types of the values this node defines.  SDNode's may
/// define multiple values simultaneously.
const EVT *ValueList;

/// List of uses for this SDNode.
SDUse *UseList = nullptr;

/// The number of entries in the Operand/Value list.
unsigned short NumOperands = 0;
unsigned short NumValues;

// The ordering of the SDNodes. It roughly corresponds to the ordering of the
// original LLVM instructions.
// This is used for turning off scheduling, because we'll forgo
// the normal scheduling algorithms and output the instructions according to
// this ordering.
unsigned IROrder;

/// Source line information.
DebugLoc debugLoc;

/// Return a pointer to the specified value type.
static const EVT *getValueTypeList(EVT VT);

SDNodeFlags Flags;

616public:
/// Unique and persistent id per SDNode in the DAG.
/// Used for debug printing.
uint16_t PersistentId;

//===--------------------------------------------------------------------===//
//  Accessors
//

/// Return the SelectionDAG opcode value for this node. For
/// pre-isel nodes (those for which isMachineOpcode returns false), these
/// are the opcode values in the ISD and <target>ISD namespaces. For
/// post-isel opcodes, see getMachineOpcode.
unsigned getOpcode()  const { return (unsigned short)NodeType; }

/// Test if this node has a target-specific opcode (in the
/// \<target\>ISD namespace).
bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }

/// Test if this node has a target-specific opcode that may raise
/// FP exceptions (in the \<target\>ISD namespace and greater than
/// FIRST_TARGET_STRICTFP_OPCODE).  Note that all target memory
/// opcode are currently automatically considered to possibly raise
/// FP exceptions as well.
bool isTargetStrictFPOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
}

/// Test if this node has a target-specific
/// memory-referencing opcode (in the \<target\>ISD namespace and
/// greater than FIRST_TARGET_MEMORY_OPCODE).
bool isTargetMemoryOpcode() const {
  return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
}

/// Return true if the type of the node type undefined.
bool isUndef() const { return NodeType == ISD::UNDEF; }
5
←
Assuming field 'NodeType' is not equal to UNDEF, which participates in a condition later→
6
←
Returning zero, which participates in a condition later→

/// Test if this node is a memory intrinsic (with valid pointer information).
/// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
/// non-memory intrinsics (with chains) that are not really instances of
/// MemSDNode. For such nodes, we need some extra state to determine the
/// proper classof relationship.
bool isMemIntrinsic() const {
  return (NodeType == ISD::INTRINSIC_W_CHAIN ||
          NodeType == ISD::INTRINSIC_VOID) &&
         SDNodeBits.IsMemIntrinsic;
}

/// Test if this node is a strict floating point pseudo-op.
bool isStrictFPOpcode() {
  switch (NodeType) {
    default:
      return false;
    case ISD::STRICT_FP16_TO_FP:
    case ISD::STRICT_FP_TO_FP16:
672#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN)               \
    case ISD::STRICT_##DAGN:
674#include "llvm/IR/ConstrainedOps.def"
      return true;
  }
}

/// Test if this node has a post-isel opcode, directly
/// corresponding to a MachineInstr opcode.
bool isMachineOpcode() const { return NodeType < 0; }

/// This may only be called if isMachineOpcode returns
/// true. It returns the MachineInstr opcode value that the node's opcode
/// corresponds to.
unsigned getMachineOpcode() const {
  assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 687, __extension__ __PRETTY_FUNCTION__));
  return ~NodeType;
}

bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }

bool isDivergent() const { return SDNodeBits.IsDivergent; }

/// Return true if there are no uses of this node.
bool use_empty() const { return UseList == nullptr; }

/// Return true if there is exactly one use of this node.
bool hasOneUse() const { return hasSingleElement(uses()); }

/// Return the number of uses of this node. This method takes
/// time proportional to the number of uses.
size_t use_size() const { return std::distance(use_begin(), use_end()); }

/// Return the unique node id.
int getNodeId() const { return NodeId; }

/// Set unique node id.
void setNodeId(int Id) { NodeId = Id; }

/// Return the node ordering.
unsigned getIROrder() const { return IROrder; }

/// Set the node ordering.
void setIROrder(unsigned Order) { IROrder = Order; }

/// Return the source location info.
const DebugLoc &getDebugLoc() const { return debugLoc; }

/// Set source location info.  Try to avoid this, putting
/// it in the constructor is preferable.
void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }

/// This class provides iterator support for SDUse
/// operands that use a specific SDNode.
class use_iterator {
  friend class SDNode;

  SDUse *Op = nullptr;

  explicit use_iterator(SDUse *op) : Op(op) {}

public:
  using iterator_category = std::forward_iterator_tag;
  using value_type = SDUse;
  using difference_type = std::ptrdiff_t;
  using pointer = value_type *;
  using reference = value_type &;

  use_iterator() = default;
  use_iterator(const use_iterator &I) : Op(I.Op) {}

  bool operator==(const use_iterator &x) const {
    return Op == x.Op;
  }
  bool operator!=(const use_iterator &x) const {
    return !operator==(x);
  }

  /// Return true if this iterator is at the end of uses list.
  bool atEnd() const { return Op == nullptr; }

  // Iterator traversal: forward iteration only.
  use_iterator &operator++() {          // Preincrement
    assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 756, __extension__ __PRETTY_FUNCTION__));
    Op = Op->getNext();
    return *this;
  }

  use_iterator operator++(int) {        // Postincrement
    use_iterator tmp = *this; ++*this; return tmp;
  }

  /// Retrieve a pointer to the current user node.
  SDNode *operator*() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 767, __extension__ __PRETTY_FUNCTION__));
    return Op->getUser();
  }

  SDNode *operator->() const { return operator*(); }

  SDUse &getUse() const { return *Op; }

  /// Retrieve the operand # of this use in its user.
  unsigned getOperandNo() const {
    assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 777, __extension__ __PRETTY_FUNCTION__));
    return (unsigned)(Op - Op->getUser()->OperandList);
  }
};

/// Provide iteration support to walk over all uses of an SDNode.
use_iterator use_begin() const {
  return use_iterator(UseList);
}

static use_iterator use_end() { return use_iterator(nullptr); }

inline iterator_range<use_iterator> uses() {
  return make_range(use_begin(), use_end());
}
inline iterator_range<use_iterator> uses() const {
  return make_range(use_begin(), use_end());
}

/// Return true if there are exactly NUSES uses of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;

/// Return true if there are any use of the indicated value.
/// This method ignores uses of other values defined by this operation.
bool hasAnyUseOfValue(unsigned Value) const;

/// Return true if this node is the only use of N.
bool isOnlyUserOf(const SDNode *N) const;

/// Return true if this node is an operand of N.
bool isOperandOf(const SDNode *N) const;

/// Return true if this node is a predecessor of N.
/// NOTE: Implemented on top of hasPredecessor and every bit as
/// expensive. Use carefully.
bool isPredecessorOf(const SDNode *N) const {
  return N->hasPredecessor(this);
}

/// Return true if N is a predecessor of this node.
/// N is either an operand of this node, or can be reached by recursively
/// traversing up the operands.
/// NOTE: This is an expensive method. Use it carefully.
bool hasPredecessor(const SDNode *N) const;

/// Returns true if N is a predecessor of any node in Worklist. This
/// helper keeps Visited and Worklist sets externally to allow unions
/// searches to be performed in parallel, caching of results across
/// queries and incremental addition to Worklist. Stops early if N is
/// found but will resume. Remember to clear Visited and Worklists
/// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
/// giving up. The TopologicalPrune flag signals that positive NodeIds are
/// topologically ordered (Operands have strictly smaller node id) and search
/// can be pruned leveraging this.
static bool hasPredecessorHelper(const SDNode *N,
                                 SmallPtrSetImpl<const SDNode *> &Visited,
                                 SmallVectorImpl<const SDNode *> &Worklist,
                                 unsigned int MaxSteps = 0,
                                 bool TopologicalPrune = false) {
  SmallVector<const SDNode *, 8> DeferredNodes;
  if (Visited.count(N))
    return true;

  // Node Id's are assigned in three places: As a topological
  // ordering (> 0), during legalization (results in values set to
  // 0), new nodes (set to -1). If N has a topolgical id then we
  // know that all nodes with ids smaller than it cannot be
  // successors and we need not check them. Filter out all node
  // that can't be matches. We add them to the worklist before exit
  // in case of multiple calls. Note that during selection the topological id
  // may be violated if a node's predecessor is selected before it. We mark
  // this at selection negating the id of unselected successors and
  // restricting topological pruning to positive ids.

  int NId = N->getNodeId();
  // If we Invalidated the Id, reconstruct original NId.
  if (NId < -1)
    NId = -(NId + 1);

  bool Found = false;
  while (!Worklist.empty()) {
    const SDNode *M = Worklist.pop_back_val();
    int MId = M->getNodeId();
    if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
        (MId > 0) && (MId < NId)) {
      DeferredNodes.push_back(M);
      continue;
    }
    for (const SDValue &OpV : M->op_values()) {
      SDNode *Op = OpV.getNode();
      if (Visited.insert(Op).second)
        Worklist.push_back(Op);
      if (Op == N)
        Found = true;
    }
    if (Found)
      break;
    if (MaxSteps != 0 && Visited.size() >= MaxSteps)
      break;
  }
  // Push deferred nodes back on worklist.
  Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
  // If we bailed early, conservatively return found.
  if (MaxSteps != 0 && Visited.size() >= MaxSteps)
    return true;
  return Found;
}

/// Return true if all the users of N are contained in Nodes.
/// NOTE: Requires at least one match, but doesn't require them all.
static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);

/// Return the number of values used by this operation.
unsigned getNumOperands() const { return NumOperands; }

/// Return the maximum number of operands that a SDNode can hold.
static constexpr size_t getMaxNumOperands() {
  return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
}

/// Helper method returns the integer value of a ConstantSDNode operand.
inline uint64_t getConstantOperandVal(unsigned Num) const;

/// Helper method returns the APInt of a ConstantSDNode operand.
inline const APInt &getConstantOperandAPInt(unsigned Num) const;

const SDValue &getOperand(unsigned Num) const {
  assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 905, __extension__ __PRETTY_FUNCTION__));
  return OperandList[Num];
}

using op_iterator = SDUse *;

op_iterator op_begin() const { return OperandList; }
op_iterator op_end() const { return OperandList+NumOperands; }
ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }

/// Iterator for directly iterating over the operand SDValue's.
struct value_op_iterator
    : iterator_adaptor_base<value_op_iterator, op_iterator,
                            std::random_access_iterator_tag, SDValue,
                            ptrdiff_t, value_op_iterator *,
                            value_op_iterator *> {
  explicit value_op_iterator(SDUse *U = nullptr)
    : iterator_adaptor_base(U) {}

  const SDValue &operator*() const { return I->get(); }
};

iterator_range<value_op_iterator> op_values() const {
  return make_range(value_op_iterator(op_begin()),
                    value_op_iterator(op_end()));
}

SDVTList getVTList() const {
  SDVTList X = { ValueList, NumValues };
  return X;
}

/// If this node has a glue operand, return the node
/// to which the glue operand points. Otherwise return NULL.
SDNode *getGluedNode() const {
  if (getNumOperands() != 0 &&
      getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
    return getOperand(getNumOperands()-1).getNode();
  return nullptr;
}

/// If this node has a glue value with a user, return
/// the user (there is at most one). Otherwise return NULL.
SDNode *getGluedUser() const {
  for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
    if (UI.getUse().get().getValueType() == MVT::Glue)
      return *UI;
  return nullptr;
}

SDNodeFlags getFlags() const { return Flags; }
void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }

/// Clear any flags in this node that aren't also set in Flags.
/// If Flags is not in a defined state then this has no effect.
void intersectFlagsWith(const SDNodeFlags Flags);

/// Return the number of values defined/returned by this operator.
unsigned getNumValues() const { return NumValues; }

/// Return the type of a specified result.
EVT getValueType(unsigned ResNo) const {
  assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 967, __extension__ __PRETTY_FUNCTION__));
  return ValueList[ResNo];
}

/// Return the type of a specified result as a simple type.
MVT getSimpleValueType(unsigned ResNo) const {
  return getValueType(ResNo).getSimpleVT();
}

/// Returns MVT::getSizeInBits(getValueType(ResNo)).
///
/// If the value type is a scalable vector type, the scalable property will
/// be set and the runtime size will be a positive integer multiple of the
/// base size.
TypeSize getValueSizeInBits(unsigned ResNo) const {
  return getValueType(ResNo).getSizeInBits();
}

using value_iterator = const EVT *;

value_iterator value_begin() const { return ValueList; }
value_iterator value_end() const { return ValueList+NumValues; }
iterator_range<value_iterator> values() const {
  return llvm::make_range(value_begin(), value_end());
}

/// Return the opcode of this operation for printing.
std::string getOperationName(const SelectionDAG *G = nullptr) const;
static const char* getIndexedModeName(ISD::MemIndexedMode AM);
void print_types(raw_ostream &OS, const SelectionDAG *G) const;
void print_details(raw_ostream &OS, const SelectionDAG *G) const;
void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and all children down to
/// the leaves.  The given SelectionDAG allows target-specific nodes
/// to be printed in human-readable form.  Unlike printr, this will
/// print the whole DAG, including children that appear multiple
/// times.
///
void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;

/// Print a SelectionDAG node and children up to
/// depth "depth."  The given SelectionDAG allows target-specific
/// nodes to be printed in human-readable form.  Unlike printr, this
/// will print children that appear multiple times wherever they are
/// used.
///
void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                     unsigned depth = 100) const;

/// Dump this node, for debugging.
void dump() const;

/// Dump (recursively) this node and its use-def subgraph.
void dumpr() const;

/// Dump this node, for debugging.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dump(const SelectionDAG *G) const;

/// Dump (recursively) this node and its use-def subgraph.
/// The given SelectionDAG allows target-specific nodes to be printed
/// in human-readable form.
void dumpr(const SelectionDAG *G) const;

/// printrFull to dbgs().  The given SelectionDAG allows
/// target-specific nodes to be printed in human-readable form.
/// Unlike dumpr, this will print the whole DAG, including children
/// that appear multiple times.
void dumprFull(const SelectionDAG *G = nullptr) const;

/// printrWithDepth to dbgs().  The given
/// SelectionDAG allows target-specific nodes to be printed in
/// human-readable form.  Unlike dumpr, this will print children
/// that appear multiple times wherever they are used.
///
void dumprWithDepth(const SelectionDAG *G = nullptr,
                    unsigned depth = 100) const;

/// Gather unique data for the node.
void Profile(FoldingSetNodeID &ID) const;

/// This method should only be used by the SDUse class.
void addUse(SDUse &U) { U.addToList(&UseList); }

1054protected:
static SDVTList getSDVTList(EVT VT) {
  SDVTList Ret = { getValueTypeList(VT), 1 };
  return Ret;
}

/// Create an SDNode.
///
/// SDNodes are created without any operands, and never own the operand
/// storage. To add operands, see SelectionDAG::createOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
    : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
      IROrder(Order), debugLoc(std::move(dl)) {
  memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
  assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
 "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1068, __extension__ __PRETTY_FUNCTION__));
  assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__))
         "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
 "NumValues wasn't wide enough for its operands!") ? void (0)
 : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__));
}

/// Release the operands and set this node to have zero operands.
void DropOperands();
1075};

1077/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1078/// into SDNode creation functions.
1079/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1080/// from the original Instruction, and IROrder is the ordinal position of
1081/// the instruction.
1082/// When an SDNode is created after the DAG is being built, both DebugLoc and
1083/// the IROrder are propagated from the original SDNode.
1084/// So SDLoc class provides two constructors besides the default one, one to
1085/// be used by the DAGBuilder, the other to be used by others.
1086class SDLoc {
1087private:
DebugLoc DL;
int IROrder = 0;

1091public:
SDLoc() = default;
SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
SDLoc(const Instruction *I, int Order) : IROrder(Order) {
  assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1096, __extension__ __PRETTY_FUNCTION__));
  if (I)
    DL = I->getDebugLoc();
}

unsigned getIROrder() const { return IROrder; }
const DebugLoc &getDebugLoc() const { return DL; }
1103};

1105// Define inline functions from the SDValue class.

1107inline SDValue::SDValue(SDNode *node, unsigned resno)
  : Node(node), ResNo(resno) {
// Explicitly check for !ResNo to avoid use-after-free, because there are
// callers that use SDValue(N, 0) with a deleted N to indicate successful
// combines.
assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__))
       "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__));
assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1114, __extension__ __PRETTY_FUNCTION__));
1115}

1117inline unsigned SDValue::getOpcode() const {
return Node->getOpcode();
1119}

1121inline EVT SDValue::getValueType() const {
return Node->getValueType(ResNo);
1123}

1125inline unsigned SDValue::getNumOperands() const {
return Node->getNumOperands();
1127}

1129inline const SDValue &SDValue::getOperand(unsigned i) const {
return Node->getOperand(i);
1131}

1133inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
return Node->getConstantOperandVal(i);
1135}

1137inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
return Node->getConstantOperandAPInt(i);
1139}

1141inline bool SDValue::isTargetOpcode() const {
return Node->isTargetOpcode();
1143}

1145inline bool SDValue::isTargetMemoryOpcode() const {
return Node->isTargetMemoryOpcode();
1147}

1149inline bool SDValue::isMachineOpcode() const {
return Node->isMachineOpcode();
1151}

1153inline unsigned SDValue::getMachineOpcode() const {
return Node->getMachineOpcode();
1155}

1157inline bool SDValue::isUndef() const {
return Node->isUndef();
4
←
Calling 'SDNode::isUndef'→
7
←
Returning from 'SDNode::isUndef'→
8
←
Returning zero, which participates in a condition later→
1159}

1161inline bool SDValue::use_empty() const {
return !Node->hasAnyUseOfValue(ResNo);
1163}

1165inline bool SDValue::hasOneUse() const {
return Node->hasNUsesOfValue(1, ResNo);
1167}

1169inline const DebugLoc &SDValue::getDebugLoc() const {
return Node->getDebugLoc();
1171}

1173inline void SDValue::dump() const {
return Node->dump();
1175}

1177inline void SDValue::dump(const SelectionDAG *G) const {
return Node->dump(G);
1179}

1181inline void SDValue::dumpr() const {
return Node->dumpr();
1183}

1185inline void SDValue::dumpr(const SelectionDAG *G) const {
return Node->dumpr(G);
1187}

1189// Define inline functions from the SDUse class.

1191inline void SDUse::set(const SDValue &V) {
if (Val.getNode()) removeFromList();
Val = V;
if (V.getNode()) V.getNode()->addUse(*this);
1195}

1197inline void SDUse::setInitial(const SDValue &V) {
Val = V;
V.getNode()->addUse(*this);
1200}

1202inline void SDUse::setNode(SDNode *N) {
if (Val.getNode()) removeFromList();
Val.setNode(N);
if (N) N->addUse(*this);
1206}

1208/// This class is used to form a handle around another node that
1209/// is persistent and is updated across invocations of replaceAllUsesWith on its
1210/// operand.  This node should be directly created by end-users and not added to
1211/// the AllNodes list.
1212class HandleSDNode : public SDNode {
SDUse Op;

1215public:
explicit HandleSDNode(SDValue X)
  : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
  // HandleSDNodes are never inserted into the DAG, so they won't be
  // auto-numbered. Use ID 65535 as a sentinel.
  PersistentId = 0xffff;

  // Manually set up the operand list. This node type is special in that it's
  // always stack allocated and SelectionDAG does not manage its operands.
  // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
  // be so special.
  Op.setUser(this);
  Op.setInitial(X);
  NumOperands = 1;
  OperandList = &Op;
}
~HandleSDNode();

const SDValue &getValue() const { return Op; }
1234};

1236class AddrSpaceCastSDNode : public SDNode {
1237private:
unsigned SrcAddrSpace;
unsigned DestAddrSpace;

1241public:
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
                    unsigned SrcAS, unsigned DestAS);

unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ADDRSPACECAST;
}
1251};

1253/// This is an abstract virtual class for memory operations.
1254class MemSDNode : public SDNode {
1255private:
// VT of in-memory value.
EVT MemoryVT;

1259protected:
/// Memory reference information.
MachineMemOperand *MMO;

1263public:
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
          EVT memvt, MachineMemOperand *MMO);

bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); }

/// Returns alignment and volatility of the memory access
Align getOriginalAlign() const { return MMO->getBaseAlign(); }
Align getAlign() const { return MMO->getAlign(); }
// FIXME: Remove once transition to getAlign is over.
unsigned getAlignment() const { return MMO->getAlign().value(); }

/// Return the SubclassData value, without HasDebugValue. This contains an
/// encoding of the volatile flag, as well as bits used by subclasses. This
/// function should only be used to compute a FoldingSetNodeID value.
/// The HasDebugValue bit is masked out because CSE map needs to match
/// nodes with debug info with nodes without debug info. Same is about
/// isDivergent bit.
unsigned getRawSubclassData() const {
  uint16_t Data;
  union {
    char RawSDNodeBits[sizeof(uint16_t)];
    SDNodeBitfields SDNodeBits;
  };
  memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
  SDNodeBits.HasDebugValue = 0;
  SDNodeBits.IsDivergent = false;
  memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
  return Data;
}

bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
bool isInvariant() const { return MemSDNodeBits.IsInvariant; }

// Returns the offset from the location of the access.
int64_t getSrcValueOffset() const { return MMO->getOffset(); }

/// Returns the AA info that describes the dereference.
AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }

/// Returns the Ranges that describes the dereference.
const MDNode *getRanges() const { return MMO->getRanges(); }

/// Returns the synchronization scope ID for this memory operation.
SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }

/// Return the atomic ordering requirements for this memory operation. For
/// cmpxchg atomic operations, return the atomic ordering requirements when
/// store occurs.
AtomicOrdering getSuccessOrdering() const {
  return MMO->getSuccessOrdering();
}

/// Return a single atomic ordering that is at least as strong as both the
/// success and failure orderings for an atomic operation.  (For operations
/// other than cmpxchg, this is equivalent to getSuccessOrdering().)
AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }

/// Return true if the memory operation ordering is Unordered or higher.
bool isAtomic() const { return MMO->isAtomic(); }

/// Returns true if the memory operation doesn't imply any ordering
/// constraints on surrounding memory operations beyond the normal memory
/// aliasing rules.
bool isUnordered() const { return MMO->isUnordered(); }

/// Returns true if the memory operation is neither atomic or volatile.
bool isSimple() const { return !isAtomic() && !isVolatile(); }

/// Return the type of the in-memory value.
EVT getMemoryVT() const { return MemoryVT; }

/// Return a MachineMemOperand object describing the memory
/// reference performed by operation.
MachineMemOperand *getMemOperand() const { return MMO; }

const MachinePointerInfo &getPointerInfo() const {
  return MMO->getPointerInfo();
}

/// Return the address space for the associated pointer
unsigned getAddressSpace() const {
  return getPointerInfo().getAddrSpace();
}

/// Update this MemSDNode's MachineMemOperand information
/// to reflect the alignment of NewMMO, if it has a greater alignment.
/// This must only be used when the new alignment applies to all users of
/// this MachineMemOperand.
void refineAlignment(const MachineMemOperand *NewMMO) {
  MMO->refineAlignment(NewMMO);
}

const SDValue &getChain() const { return getOperand(0); }

const SDValue &getBasePtr() const {
  switch (getOpcode()) {
  case ISD::STORE:
  case ISD::VP_STORE:
  case ISD::MSTORE:
  case ISD::VP_SCATTER:
    return getOperand(2);
  case ISD::MGATHER:
  case ISD::MSCATTER:
    return getOperand(3);
  default:
    return getOperand(1);
  }
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // For some targets, we lower some target intrinsics to a MemIntrinsicNode
  // with either an intrinsic or a target opcode.
  switch (N->getOpcode()) {
  case ISD::LOAD:
  case ISD::STORE:
  case ISD::PREFETCH:
  case ISD::ATOMIC_CMP_SWAP:
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
  case ISD::ATOMIC_SWAP:
  case ISD::ATOMIC_LOAD_ADD:
  case ISD::ATOMIC_LOAD_SUB:
  case ISD::ATOMIC_LOAD_AND:
  case ISD::ATOMIC_LOAD_CLR:
  case ISD::ATOMIC_LOAD_OR:
  case ISD::ATOMIC_LOAD_XOR:
  case ISD::ATOMIC_LOAD_NAND:
  case ISD::ATOMIC_LOAD_MIN:
  case ISD::ATOMIC_LOAD_MAX:
  case ISD::ATOMIC_LOAD_UMIN:
  case ISD::ATOMIC_LOAD_UMAX:
  case ISD::ATOMIC_LOAD_FADD:
  case ISD::ATOMIC_LOAD_FSUB:
  case ISD::ATOMIC_LOAD:
  case ISD::ATOMIC_STORE:
  case ISD::MLOAD:
  case ISD::MSTORE:
  case ISD::MGATHER:
  case ISD::MSCATTER:
  case ISD::VP_LOAD:
  case ISD::VP_STORE:
  case ISD::VP_GATHER:
  case ISD::VP_SCATTER:
    return true;
  default:
    return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
  }
}
1415};

1417/// This is an SDNode representing atomic operations.
1418class AtomicSDNode : public MemSDNode {
1419public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
             EVT MemVT, MachineMemOperand *MMO)
  : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
  assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__))
          MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
 Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__));
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getVal() const { return getOperand(2); }

/// Returns true if this SDNode represents cmpxchg atomic operation, false
/// otherwise.
bool isCompareAndSwap() const {
  unsigned Op = getOpcode();
  return Op == ISD::ATOMIC_CMP_SWAP ||
         Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
}

/// For cmpxchg atomic operations, return the atomic ordering requirements
/// when store does not occur.
AtomicOrdering getFailureOrdering() const {
  assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1441, __extension__ __PRETTY_FUNCTION__));
  return MMO->getFailureOrdering();
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
         N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
         N->getOpcode() == ISD::ATOMIC_SWAP         ||
         N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_AND     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_CLR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_OR      ||
         N->getOpcode() == ISD::ATOMIC_LOAD_XOR     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_NAND    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MIN     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
         N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
         N->getOpcode() == ISD::ATOMIC_LOAD         ||
         N->getOpcode() == ISD::ATOMIC_STORE;
}
1466};

1468/// This SDNode is used for target intrinsics that touch
1469/// memory and need an associated MachineMemOperand. Its opcode may be
1470/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1471/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1472class MemIntrinsicSDNode : public MemSDNode {
1473public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
                   SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
    : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
  SDNodeBits.IsMemIntrinsic = true;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  // We lower some target intrinsics to their target opcode
  // early a node with a target opcode can be of this class
  return N->isMemIntrinsic()             ||
         N->getOpcode() == ISD::PREFETCH ||
         N->isTargetMemoryOpcode();
}
1488};

1490/// This SDNode is used to implement the code generator
1491/// support for the llvm IR shufflevector instruction.  It combines elements
1492/// from two input vectors into a new input vector, with the selection and
1493/// ordering of elements determined by an array of integers, referred to as
1494/// the shuffle mask.  For input vectors of width N, mask indices of 0..N-1
1495/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1496/// An index of -1 is treated as undef, such that the code generator may put
1497/// any value in the corresponding element of the result.
1498class ShuffleVectorSDNode : public SDNode {
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed.
const int *Mask;

1503protected:
friend class SelectionDAG;

ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
    : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}

1509public:
ArrayRef<int> getMask() const {
  EVT VT = getValueType(0);
  return makeArrayRef(Mask, VT.getVectorNumElements());
}

int getMaskElt(unsigned Idx) const {
  assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
 ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1516, __extension__ __PRETTY_FUNCTION__));
  return Mask[Idx];
}

bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }

int getSplatIndex() const {
  assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1523, __extension__ __PRETTY_FUNCTION__));
  EVT VT = getValueType(0);
  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
    if (Mask[i] >= 0)
      return Mask[i];

  // We can choose any index value here and be correct because all elements
  // are undefined. Return 0 for better potential for callers to simplify.
  return 0;
}

static bool isSplatMask(const int *Mask, EVT VT);

/// Change values in a shuffle permute mask assuming
/// the two vector operands have swapped position.
static void commuteMask(MutableArrayRef<int> Mask) {
  unsigned NumElems = Mask.size();
  for (unsigned i = 0; i != NumElems; ++i) {
    int idx = Mask[i];
    if (idx < 0)
      continue;
    else if (idx < (int)NumElems)
      Mask[i] = idx + NumElems;
    else
      Mask[i] = idx - NumElems;
  }
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VECTOR_SHUFFLE;
}
1554};

1556class ConstantSDNode : public SDNode {
friend class SelectionDAG;

const ConstantInt *Value;

ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
             getSDVTList(VT)),
      Value(val) {
  ConstantSDNodeBits.IsOpaque = isOpaque;
}

1568public:
const ConstantInt *getConstantIntValue() const { return Value; }
const APInt &getAPIntValue() const { return Value->getValue(); }
uint64_t getZExtValue() const { return Value->getZExtValue(); }
int64_t getSExtValue() const { return Value->getSExtValue(); }
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
  return Value->getLimitedValue(Limit);
}
MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
Align getAlignValue() const { return Value->getAlignValue(); }

bool isOne() const { return Value->isOne(); }
bool isZero() const { return Value->isZero(); }
// NOTE: This is soft-deprecated.  Please use `isZero()` instead.
bool isNullValue() const { return isZero(); }
bool isAllOnes() const { return Value->isMinusOne(); }
// NOTE: This is soft-deprecated.  Please use `isAllOnes()` instead.
bool isAllOnesValue() const { return isAllOnes(); }
bool isMaxSignedValue() const { return Value->isMaxValue(true); }
bool isMinSignedValue() const { return Value->isMinValue(true); }

bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Constant ||
         N->getOpcode() == ISD::TargetConstant;
}
1595};

1597uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1599}

1601const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1603}

1605class ConstantFPSDNode : public SDNode {
friend class SelectionDAG;

const ConstantFP *Value;

ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
    : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
             DebugLoc(), getSDVTList(VT)),
      Value(val) {}

1615public:
const APFloat& getValueAPF() const { return Value->getValueAPF(); }
const ConstantFP *getConstantFPValue() const { return Value; }

/// Return true if the value is positive or negative zero.
bool isZero() const { return Value->isZero(); }

/// Return true if the value is a NaN.
bool isNaN() const { return Value->isNaN(); }

/// Return true if the value is an infinity
bool isInfinity() const { return Value->isInfinity(); }

/// Return true if the value is negative.
bool isNegative() const { return Value->isNegative(); }

/// We don't rely on operator== working on double values, as
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
/// As such, this method can be used to do an exact bit-for-bit comparison of
/// two floating point values.

/// We leave the version with the double argument here because it's just so
/// convenient to write "2.0" and the like.  Without this function we'd
/// have to duplicate its logic everywhere it's called.
bool isExactlyValue(double V) const {
  return Value->getValueAPF().isExactlyValue(V);
}
bool isExactlyValue(const APFloat& V) const;

static bool isValueValidForType(EVT VT, const APFloat& Val);

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantFP ||
         N->getOpcode() == ISD::TargetConstantFP;
}
1650};

1652/// Returns true if \p V is a constant integer zero.
1653bool isNullConstant(SDValue V);

1655/// Returns true if \p V is an FP constant with a value of positive zero.
1656bool isNullFPConstant(SDValue V);

1658/// Returns true if \p V is an integer constant with all bits set.
1659bool isAllOnesConstant(SDValue V);

1661/// Returns true if \p V is a constant integer one.
1662bool isOneConstant(SDValue V);

1664/// Return the non-bitcasted source operand of \p V if it exists.
1665/// If \p V is not a bitcasted value, it is returned as-is.
1666SDValue peekThroughBitcasts(SDValue V);

1668/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1669/// If \p V is not a bitcasted one-use value, it is returned as-is.
1670SDValue peekThroughOneUseBitcasts(SDValue V);

1672/// Return the non-extracted vector source operand of \p V if it exists.
1673/// If \p V is not an extracted subvector, it is returned as-is.
1674SDValue peekThroughExtractSubvectors(SDValue V);

1676/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1677/// constant is canonicalized to be operand 1.
1678bool isBitwiseNot(SDValue V, bool AllowUndefs = false);

1680/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1681ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1684/// Returns the SDNode if it is a demanded constant splat BuildVector or
1685/// constant int.
1686ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
                                  bool AllowUndefs = false,
                                  bool AllowTruncation = false);

1690/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1691ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);

1693/// Returns the SDNode if it is a demanded constant splat BuildVector or
1694/// constant float.
1695ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
                                      bool AllowUndefs = false);

1698/// Return true if the value is a constant 0 integer or a splatted vector of
1699/// a constant 0 integer (with no undefs by default).
1700/// Build vector implicit truncation is not an issue for null values.
1701bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);

1703/// Return true if the value is a constant 1 integer or a splatted vector of a
1704/// constant 1 integer (with no undefs).
1705/// Does not permit build vector implicit truncation.
1706bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);

1708/// Return true if the value is a constant -1 integer or a splatted vector of a
1709/// constant -1 integer (with no undefs).
1710/// Does not permit build vector implicit truncation.
1711bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);

1713/// Return true if \p V is either a integer or FP constant.
1714inline bool isIntOrFPConstant(SDValue V) {
return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1716}

1718class GlobalAddressSDNode : public SDNode {
friend class SelectionDAG;

const GlobalValue *TheGlobal;
int64_t Offset;
unsigned TargetFlags;

GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
                    const GlobalValue *GA, EVT VT, int64_t o,
                    unsigned TF);

1729public:
const GlobalValue *getGlobal() const { return TheGlobal; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }
// Return the address space this GlobalAddress belongs to.
unsigned getAddressSpace() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::GlobalAddress ||
         N->getOpcode() == ISD::TargetGlobalAddress ||
         N->getOpcode() == ISD::GlobalTLSAddress ||
         N->getOpcode() == ISD::TargetGlobalTLSAddress;
}
1742};

1744class FrameIndexSDNode : public SDNode {
friend class SelectionDAG;

int FI;

FrameIndexSDNode(int fi, EVT VT, bool isTarg)
  : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
    0, DebugLoc(), getSDVTList(VT)), FI(fi) {
}

1754public:
int getIndex() const { return FI; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::FrameIndex ||
         N->getOpcode() == ISD::TargetFrameIndex;
}
1761};

1763/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1764/// the offet and size that are started/ended in the underlying FrameIndex.
1765class LifetimeSDNode : public SDNode {
friend class SelectionDAG;
int64_t Size;
int64_t Offset; // -1 if offset is unknown.

LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
               SDVTList VTs, int64_t Size, int64_t Offset)
    : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1773public:
int64_t getFrameIndex() const {
  return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
}

bool hasOffset() const { return Offset >= 0; }
int64_t getOffset() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1780, __extension__ __PRETTY_FUNCTION__));
  return Offset;
}
int64_t getSize() const {
  assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1784, __extension__ __PRETTY_FUNCTION__));
  return Size;
}

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LIFETIME_START ||
         N->getOpcode() == ISD::LIFETIME_END;
}
1793};

1795/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1796/// the index of the basic block being probed. A pseudo probe serves as a place
1797/// holder and will be removed at the end of compilation. It does not have any
1798/// operand because we do not want the instruction selection to deal with any.
1799class PseudoProbeSDNode : public SDNode {
friend class SelectionDAG;
uint64_t Guid;
uint64_t Index;
uint32_t Attributes;

PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
                  SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
    : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
      Attributes(Attr) {}

1810public:
uint64_t getGuid() const { return Guid; }
uint64_t getIndex() const { return Index; }
uint32_t getAttributes() const { return Attributes; }

// Methods to support isa and dyn_cast
static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::PSEUDO_PROBE;
}
1819};

1821class JumpTableSDNode : public SDNode {
friend class SelectionDAG;

int JTI;
unsigned TargetFlags;

JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
  : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
    0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
}

1832public:
int getIndex() const { return JTI; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::JumpTable ||
         N->getOpcode() == ISD::TargetJumpTable;
}
1840};

1842class ConstantPoolSDNode : public SDNode {
friend class SelectionDAG;

union {
  const Constant *ConstVal;
  MachineConstantPoolValue *MachineCPVal;
} Val;
int Offset;  // It's a MachineConstantPoolValue if top bit is set.
Align Alignment; // Minimum alignment requirement of CP.
unsigned TargetFlags;

ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1858, __extension__ __PRETTY_FUNCTION__));
  Val.ConstVal = c;
}

ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
                   Align Alignment, unsigned TF)
    : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
             DebugLoc(), getSDVTList(VT)),
      Offset(o), Alignment(Alignment), TargetFlags(TF) {
  assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1867, __extension__ __PRETTY_FUNCTION__));
  Val.MachineCPVal = v;
  Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
}

1872public:
bool isMachineConstantPoolEntry() const {
  return Offset < 0;
}

const Constant *getConstVal() const {
  assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1878, __extension__ __PRETTY_FUNCTION__));
  return Val.ConstVal;
}

MachineConstantPoolValue *getMachineCPVal() const {
  assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
 "Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1883, __extension__ __PRETTY_FUNCTION__));
  return Val.MachineCPVal;
}

int getOffset() const {
  return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
}

// Return the alignment of this constant pool object, which is either 0 (for
// default alignment) or the desired value.
Align getAlign() const { return Alignment; }
unsigned getTargetFlags() const { return TargetFlags; }

Type *getType() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ConstantPool ||
         N->getOpcode() == ISD::TargetConstantPool;
}
1902};

1904/// Completely target-dependent object reference.
1905class TargetIndexSDNode : public SDNode {
friend class SelectionDAG;

unsigned TargetFlags;
int Index;
int64_t Offset;

1912public:
TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
    : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
      TargetFlags(TF), Index(Idx), Offset(Ofs) {}

unsigned getTargetFlags() const { return TargetFlags; }
int getIndex() const { return Index; }
int64_t getOffset() const { return Offset; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::TargetIndex;
}
1924};

1926class BasicBlockSDNode : public SDNode {
friend class SelectionDAG;

MachineBasicBlock *MBB;

/// Debug info is meaningful and potentially useful here, but we create
/// blocks out of order when they're jumped to, which makes it a bit
/// harder.  Let's see if we need it first.
explicit BasicBlockSDNode(MachineBasicBlock *mbb)
  : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
{}

1938public:
MachineBasicBlock *getBasicBlock() const { return MBB; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BasicBlock;
}
1944};

1946/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1947class BuildVectorSDNode : public SDNode {
1948public:
// These are constructed as SDNodes and then cast to BuildVectorSDNodes.
explicit BuildVectorSDNode() = delete;

/// Check if this is a constant splat, and if so, find the
/// smallest element size that splats the vector.  If MinSplatBits is
/// nonzero, the element size must be at least that large.  Note that the
/// splat element may be the entire vector (i.e., a one element vector).
/// Returns the splat element value in SplatValue.  Any undefined bits in
/// that value are zero, and the corresponding bits in the SplatUndef mask
/// are set.  The SplatBitSize value is set to the splat element size in
/// bits.  HasAnyUndefs is set to true if any bits in the vector are
/// undefined.  isBigEndian describes the endianness of the target.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                     unsigned &SplatBitSize, bool &HasAnyUndefs,
                     unsigned MinSplatBits = 0,
                     bool isBigEndian = false) const;

/// Returns the demanded splatted value or a null value if this is not a
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(const APInt &DemandedElts,
                      BitVector *UndefElements = nullptr) const;

/// Returns the splatted value or a null value if this is not a splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
SDValue getSplatValue(BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// The DemandedElts mask indicates the elements that must be present,
/// undemanded elements in Sequence may be null (SDValue()). If passed a
/// non-null UndefElements bitvector, it will resize it to match the original
/// vector width and set the bits where elements are undef. If result is
/// false, Sequence will be empty.
bool getRepeatedSequence(const APInt &DemandedElts,
                         SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Find the shortest repeating sequence of values in the build vector.
///
/// e.g. { u, X, u, X, u, u, X, u } -> { X }
///      { X, Y, u, Y, u, u, X, u } -> { X, Y }
///
/// Currently this must be a power-of-2 build vector.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the original vector width and set the bits where elements are undef.
/// If result is false, Sequence will be empty.
bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
                         BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant or null if this is not a constant
/// splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(const APInt &DemandedElts,
                     BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant or null if this is not a constant
/// splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantSDNode *
getConstantSplatNode(BitVector *UndefElements = nullptr) const;

/// Returns the demanded splatted constant FP or null if this is not a
/// constant FP splat.
///
/// The DemandedElts mask indicates the elements that must be in the splat.
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(const APInt &DemandedElts,
                       BitVector *UndefElements = nullptr) const;

/// Returns the splatted constant FP or null if this is not a constant
/// FP splat.
///
/// If passed a non-null UndefElements bitvector, it will resize it to match
/// the vector width and set the bits where elements are undef.
ConstantFPSDNode *
getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;

/// If this is a constant FP splat and the splatted constant FP is an
/// exact power or 2, return the log base 2 integer value.  Otherwise,
/// return -1.
///
/// The BitWidth specifies the necessary bit precision.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
                                        uint32_t BitWidth) const;

/// Extract the raw bit data from a build vector of Undef, Constant or
/// ConstantFP node elements. Each raw bit element will be \p
/// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
/// undefined elements are flagged in \p UndefElements.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
                        SmallVectorImpl<APInt> &RawBitElements,
                        BitVector &UndefElements) const;

bool isConstant() const;

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BUILD_VECTOR;
}
2065};

2067/// An SDNode that holds an arbitrary LLVM IR Value. This is
2068/// used when the SelectionDAG needs to make a simple reference to something
2069/// in the LLVM IR representation.
2070///
2071class SrcValueSDNode : public SDNode {
friend class SelectionDAG;

const Value *V;

/// Create a SrcValue for a general value.
explicit SrcValueSDNode(const Value *v)
  : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}

2080public:
/// Return the contained Value.
const Value *getValue() const { return V; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::SRCVALUE;
}
2087};

2089class MDNodeSDNode : public SDNode {
friend class SelectionDAG;

const MDNode *MD;

explicit MDNodeSDNode(const MDNode *md)
: SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
{}

2098public:
const MDNode *getMD() const { return MD; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MDNODE_SDNODE;
}
2104};

2106class RegisterSDNode : public SDNode {
friend class SelectionDAG;

Register Reg;

RegisterSDNode(Register reg, EVT VT)
  : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}

2114public:
Register getReg() const { return Reg; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::Register;
}
2120};

2122class RegisterMaskSDNode : public SDNode {
friend class SelectionDAG;

// The memory for RegMask is not owned by the node.
const uint32_t *RegMask;

RegisterMaskSDNode(const uint32_t *mask)
  : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
    RegMask(mask) {}

2132public:
const uint32_t *getRegMask() const { return RegMask; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::RegisterMask;
}
2138};

2140class BlockAddressSDNode : public SDNode {
friend class SelectionDAG;

const BlockAddress *BA;
int64_t Offset;
unsigned TargetFlags;

BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
                   int64_t o, unsigned Flags)
  : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
           BA(ba), Offset(o), TargetFlags(Flags) {}

2152public:
const BlockAddress *getBlockAddress() const { return BA; }
int64_t getOffset() const { return Offset; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::BlockAddress ||
         N->getOpcode() == ISD::TargetBlockAddress;
}
2161};

2163class LabelSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Label;

LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
    : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
  assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
 "not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2170, __extension__ __PRETTY_FUNCTION__));
}

2173public:
MCSymbol *getLabel() const { return Label; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::EH_LABEL ||
         N->getOpcode() == ISD::ANNOTATION_LABEL;
}
2180};

2182class ExternalSymbolSDNode : public SDNode {
friend class SelectionDAG;

const char *Symbol;
unsigned TargetFlags;

ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
    : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
             DebugLoc(), getSDVTList(VT)),
      Symbol(Sym), TargetFlags(TF) {}

2193public:
const char *getSymbol() const { return Symbol; }
unsigned getTargetFlags() const { return TargetFlags; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::ExternalSymbol ||
         N->getOpcode() == ISD::TargetExternalSymbol;
}
2201};

2203class MCSymbolSDNode : public SDNode {
friend class SelectionDAG;

MCSymbol *Symbol;

MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
    : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}

2211public:
MCSymbol *getMCSymbol() const { return Symbol; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MCSymbol;
}
2217};

2219class CondCodeSDNode : public SDNode {
friend class SelectionDAG;

ISD::CondCode Condition;

explicit CondCodeSDNode(ISD::CondCode Cond)
  : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    Condition(Cond) {}

2228public:
ISD::CondCode get() const { return Condition; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::CONDCODE;
}
2234};

2236/// This class is used to represent EVT's, which are used
2237/// to parameterize some operations.
2238class VTSDNode : public SDNode {
friend class SelectionDAG;

EVT ValueType;

explicit VTSDNode(EVT VT)
  : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
    ValueType(VT) {}

2247public:
EVT getVT() const { return ValueType; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VALUETYPE;
}
2253};

2255/// Base class for LoadSDNode and StoreSDNode
2256class LSBaseSDNode : public MemSDNode {
2257public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
             SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
             MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2263, __extension__ __PRETTY_FUNCTION__));
}

const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD ||
         N->getOpcode() == ISD::STORE;
}
2286};

2288/// This class is used to represent ISD::LOAD nodes.
2289class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG;

LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
           ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
           MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2297, __extension__ __PRETTY_FUNCTION__));
  assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2298, __extension__ __PRETTY_FUNCTION__));
}

2301public:
/// Return whether this is a plain node,
/// or one of the varieties of value-extending loads.
ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::LOAD;
}
2314};

2316/// This class is used to represent ISD::STORE nodes.
2317class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG;

StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
            ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
            MachineMemOperand *MMO)
    : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2325, __extension__ __PRETTY_FUNCTION__));
  assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2326, __extension__ __PRETTY_FUNCTION__));
}

2329public:
/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
void setTruncatingStore(bool Truncating) {
  StoreSDNodeBits.IsTruncating = Truncating;
}

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::STORE;
}
2345};

2347/// This base class is used to represent VP_LOAD and VP_STORE nodes
2348class VPLoadStoreSDNode : public MemSDNode {
2349public:
friend class SelectionDAG;

VPLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
                  SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
                  MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2357, __extension__ __PRETTY_FUNCTION__));
}

// VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
// VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
// Mask is a vector of i1 elements;
// the type of EVL is TLI.getVPExplicitVectorLengthTy().
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 2 : 3);
}
const SDValue &getBasePtr() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 1 : 2);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 3 : 4);
}
const SDValue &getVectorLength() const {
  return getOperand(getOpcode() == ISD::VP_LOAD ? 4 : 5);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
}
2392};

2394/// This class is used to represent a VP_LOAD node
2395class VPLoadSDNode : public VPLoadStoreSDNode {
2396public:
friend class SelectionDAG;

VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
             ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
             EVT MemVT, MachineMemOperand *MMO)
    : VPLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = isExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getVectorLength() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_LOAD;
}
bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2420};

2422/// This class is used to represent a VP_STORE node
2423class VPStoreSDNode : public VPLoadStoreSDNode {
2424public:
friend class SelectionDAG;

VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
              ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
              EVT MemVT, MachineMemOperand *MMO)
    : VPLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if this is a truncating store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }
const SDValue &getVectorLength() const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_STORE;
}
2455};

2457/// This base class is used to represent MLOAD and MSTORE nodes
2458class MaskedLoadStoreSDNode : public MemSDNode {
2459public:
friend class SelectionDAG;

MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs,
                      ISD::MemIndexedMode AM, EVT MemVT,
                      MachineMemOperand *MMO)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = AM;
  assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
 "Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2468, __extension__ __PRETTY_FUNCTION__));
}

// MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
// MaskedStoreSDNode (Chain, data, ptr, offset, mask)
// Mask is a vector of i1 elements
const SDValue &getOffset() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
}
const SDValue &getMask() const {
  return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
}

/// Return the addressing mode for this load or store:
/// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
ISD::MemIndexedMode getAddressingMode() const {
  return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
}

/// Return true if this is a pre/post inc/dec load/store.
bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }

/// Return true if this is NOT a pre/post inc/dec load/store.
bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD ||
         N->getOpcode() == ISD::MSTORE;
}
2497};

2499/// This class is used to represent an MLOAD node
2500class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2501public:
friend class SelectionDAG;

MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
                 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
  LoadSDNodeBits.ExtTy = ETy;
  LoadSDNodeBits.IsExpanding = IsExpanding;
}

ISD::LoadExtType getExtensionType() const {
  return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
}

const SDValue &getBasePtr() const { return getOperand(1); }
const SDValue &getOffset() const { return getOperand(2); }
const SDValue &getMask() const { return getOperand(3); }
const SDValue &getPassThru() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MLOAD;
}

bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2526};

2528/// This class is used to represent an MSTORE node
2529class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2530public:
friend class SelectionDAG;

MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                  ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
                  EVT MemVT, MachineMemOperand *MMO)
    : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
  StoreSDNodeBits.IsTruncating = isTrunc;
  StoreSDNodeBits.IsCompressing = isCompressing;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

/// Returns true if the op does a compression to the vector before storing.
/// The node contiguously stores the active elements (integers or floats)
/// in src (those with their respective bit set in writemask k) to unaligned
/// memory at base_addr.
bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }

const SDValue &getValue() const { return getOperand(1); }
const SDValue &getBasePtr() const { return getOperand(2); }
const SDValue &getOffset() const { return getOperand(3); }
const SDValue &getMask() const { return getOperand(4); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSTORE;
}
2560};

2562/// This is a base class used to represent
2563/// VP_GATHER and VP_SCATTER nodes
2564///
2565class VPGatherScatterSDNode : public MemSDNode {
2566public:
friend class SelectionDAG;

VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                      const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                      MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2574, __extension__ __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// VPGatherSDNode  (Chain, base, index, scale, mask, vlen)
// VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
}
const SDValue &getIndex() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
}
const SDValue &getScale() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
}
const SDValue &getMask() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
}
const SDValue &getVectorLength() const {
  return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER ||
         N->getOpcode() == ISD::VP_SCATTER;
}
2614};

2616/// This class is used to represent an VP_GATHER node
2617///
2618class VPGatherSDNode : public VPGatherScatterSDNode {
2619public:
friend class SelectionDAG;

VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
               MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_GATHER;
}
2630};

2632/// This class is used to represent an VP_SCATTER node
2633///
2634class VPScatterSDNode : public VPGatherScatterSDNode {
2635public:
friend class SelectionDAG;

VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
                            IndexType) {}

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::VP_SCATTER;
}
2648};

2650/// This is a base class used to represent
2651/// MGATHER and MSCATTER nodes
2652///
2653class MaskedGatherScatterSDNode : public MemSDNode {
2654public:
friend class SelectionDAG;

MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
                          const DebugLoc &dl, SDVTList VTs, EVT MemVT,
                          MachineMemOperand *MMO, ISD::MemIndexType IndexType)
    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
  assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
 "Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2662, __extension__ __PRETTY_FUNCTION__));
}

/// How is Index applied to BasePtr when computing addresses.
ISD::MemIndexType getIndexType() const {
  return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
}
void setIndexType(ISD::MemIndexType IndexType) {
  LSBaseSDNodeBits.AddressingMode = IndexType;
}
bool isIndexScaled() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::UNSIGNED_SCALED);
}
bool isIndexSigned() const {
  return (getIndexType() == ISD::SIGNED_SCALED) ||
         (getIndexType() == ISD::SIGNED_UNSCALED);
}

// In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode  (Chain, passthru, mask, base, index, scale)
// MaskedScatterSDNode (Chain, value, mask, base, index, scale)
// Mask is a vector of i1 elements
const SDValue &getBasePtr() const { return getOperand(3); }
const SDValue &getIndex()   const { return getOperand(4); }
const SDValue &getMask()    const { return getOperand(2); }
const SDValue &getScale()   const { return getOperand(5); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER ||
         N->getOpcode() == ISD::MSCATTER;
}
2694};

2696/// This class is used to represent an MGATHER node
2697///
2698class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2699public:
friend class SelectionDAG;

MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                   EVT MemVT, MachineMemOperand *MMO,
                   ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  LoadSDNodeBits.ExtTy = ETy;
}

const SDValue &getPassThru() const { return getOperand(1); }

ISD::LoadExtType getExtensionType() const {
  return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
}

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MGATHER;
}
2719};

2721/// This class is used to represent an MSCATTER node
2722///
2723class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2724public:
friend class SelectionDAG;

MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                    EVT MemVT, MachineMemOperand *MMO,
                    ISD::MemIndexType IndexType, bool IsTrunc)
    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
                                IndexType) {
  StoreSDNodeBits.IsTruncating = IsTrunc;
}

/// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result.
/// For floats, it is the same as doing an FP_ROUND and storing the result.
bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }

const SDValue &getValue() const { return getOperand(1); }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::MSCATTER;
}
2745};

2747/// An SDNode that represents everything that will be needed
2748/// to construct a MachineInstr. These nodes are created during the
2749/// instruction selection proper phase.
2750///
2751/// Note that the only supported way to set the `memoperands` is by calling the
2752/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2753/// inside the DAG rather than in the node.
2754class MachineSDNode : public SDNode {
2755private:
friend class SelectionDAG;

MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
    : SDNode(Opc, Order, DL, VTs) {}

// We use a pointer union between a single `MachineMemOperand` pointer and
// a pointer to an array of `MachineMemOperand` pointers. This is null when
// the number of these is zero, the single pointer variant used when the
// number is one, and the array is used for larger numbers.
//
// The array is allocated via the `SelectionDAG`'s allocator and so will
// always live until the DAG is cleaned up and doesn't require ownership here.
//
// We can't use something simpler like `TinyPtrVector` here because `SDNode`
// subclasses aren't managed in a conforming C++ manner. See the comments on
// `SelectionDAG::MorphNodeTo` which details what all goes on, but the
// constraint here is that these don't manage memory with their constructor or
// destructor and can be initialized to a good state even if they start off
// uninitialized.
PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};

// Note that this could be folded into the above `MemRefs` member if doing so
// is advantageous at some point. We don't need to store this in most cases.
// However, at the moment this doesn't appear to make the allocation any
// smaller and makes the code somewhat simpler to read.
int NumMemRefs = 0;

2783public:
using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;

ArrayRef<MachineMemOperand *> memoperands() const {
  // Special case the common cases.
  if (NumMemRefs == 0)
    return {};
  if (NumMemRefs == 1)
    return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);

  // Otherwise we have an actual array.
  return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
}
mmo_iterator memoperands_begin() const { return memoperands().begin(); }
mmo_iterator memoperands_end() const { return memoperands().end(); }
bool memoperands_empty() const { return memoperands().empty(); }

/// Clear out the memory reference descriptor list.
void clearMemRefs() {
  MemRefs = nullptr;
  NumMemRefs = 0;
}

static bool classof(const SDNode *N) {
  return N->isMachineOpcode();
}
2809};

2811/// An SDNode that records if a register contains a value that is guaranteed to
2812/// be aligned accordingly.
2813class AssertAlignSDNode : public SDNode {
Align Alignment;

2816public:
AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
    : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}

Align getAlign() const { return Alignment; }

static bool classof(const SDNode *N) {
  return N->getOpcode() == ISD::AssertAlign;
}
2825};

2827class SDNodeIterator {
const SDNode *Node;
unsigned Operand;

SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}

2833public:
using iterator_category = std::forward_iterator_tag;
using value_type = SDNode;
using difference_type = std::ptrdiff_t;
using pointer = value_type *;
using reference = value_type &;

bool operator==(const SDNodeIterator& x) const {
  return Operand == x.Operand;
}
bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }

pointer operator*() const {
  return Node->getOperand(Operand).getNode();
}
pointer operator->() const { return operator*(); }

SDNodeIterator& operator++() {                // Preincrement
  ++Operand;
  return *this;
}
SDNodeIterator operator++(int) { // Postincrement
  SDNodeIterator tmp = *this; ++*this; return tmp;
}
size_t operator-(SDNodeIterator Other) const {
  assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2859, __extension__ __PRETTY_FUNCTION__))
         "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2859, __extension__ __PRETTY_FUNCTION__));
  return Operand - Other.Operand;
}

static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
static SDNodeIterator end  (const SDNode *N) {
  return SDNodeIterator(N, N->getNumOperands());
}

unsigned getOperand() const { return Operand; }
const SDNode *getNode() const { return Node; }
2870};

2872template <> struct GraphTraits<SDNode*> {
using NodeRef = SDNode *;
using ChildIteratorType = SDNodeIterator;

static NodeRef getEntryNode(SDNode *N) { return N; }

static ChildIteratorType child_begin(NodeRef N) {
  return SDNodeIterator::begin(N);
}

static ChildIteratorType child_end(NodeRef N) {
  return SDNodeIterator::end(N);
}
2885};

2887/// A representation of the largest SDNode, for use in sizeof().
2888///
2889/// This needs to be a union because the largest node differs on 32 bit systems
2890/// with 4 and 8 byte pointer alignment, respectively.
2891using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
                                          BlockAddressSDNode,
                                          GlobalAddressSDNode,
                                          PseudoProbeSDNode>;

2896/// The SDNode class with the greatest alignment requirement.
2897using MostAlignedSDNode = GlobalAddressSDNode;

2899namespace ISD {

/// Returns true if the specified node is a non-extending and unindexed load.
inline bool isNormalLoad(const SDNode *N) {
  const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
  return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
    Ld->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-extending load.
inline bool isNON_EXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
}

/// Returns true if the specified node is a EXTLOAD.
inline bool isEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
}

/// Returns true if the specified node is a SEXTLOAD.
inline bool isSEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
}

/// Returns true if the specified node is a ZEXTLOAD.
inline bool isZEXTLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
}

/// Returns true if the specified node is an unindexed load.
inline bool isUNINDEXEDLoad(const SDNode *N) {
  return isa<LoadSDNode>(N) &&
    cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is a non-truncating
/// and unindexed store.
inline bool isNormalStore(const SDNode *N) {
  const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
  return St && !St->isTruncatingStore() &&
    St->getAddressingMode() == ISD::UNINDEXED;
}

/// Returns true if the specified node is an unindexed store.
inline bool isUNINDEXEDStore(const SDNode *N) {
  return isa<StoreSDNode>(N) &&
    cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
}

/// Attempt to match a unary predicate against a scalar/splat constant or
/// every element of a constant BUILD_VECTOR.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
bool matchUnaryPredicate(SDValue Op,
                         std::function<bool(ConstantSDNode *)> Match,
                         bool AllowUndefs = false);

/// Attempt to match a binary predicate against a pair of scalar/splat
/// constants or every element of a pair of constant BUILD_VECTORs.
/// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
/// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
bool matchBinaryPredicate(
    SDValue LHS, SDValue RHS,
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
    bool AllowUndefs = false, bool AllowTypeMismatch = false);

/// Returns true if the specified value is the overflow result from one
/// of the overflow intrinsic nodes.
inline bool isOverflowIntrOpRes(SDValue Op) {
  unsigned Opc = Op.getOpcode();
  return (Op.getResNo() == 1 &&
          (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
           Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
}

2977} // end namespace ISD

2979} // end namespace llvm

2981#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H

←

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h

→

1//===-- llvm/ADT/APInt.h - For Arbitrary Precision Integer -----*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a class to represent arbitrary precision
11/// integral constant values and operations on them.
12///
13//===----------------------------------------------------------------------===//

15#ifndef LLVM_ADT_APINT_H
16#define LLVM_ADT_APINT_H

18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/MathExtras.h"
20#include <cassert>
21#include <climits>
22#include <cstring>
23#include <utility>

25namespace llvm {
26class FoldingSetNodeID;
27class StringRef;
28class hash_code;
29class raw_ostream;

31template <typename T> class SmallVectorImpl;
32template <typename T> class ArrayRef;
33template <typename T> class Optional;
34template <typename T> struct DenseMapInfo;

36class APInt;

38inline APInt operator-(APInt);

40//===----------------------------------------------------------------------===//
41//                              APInt Class
42//===----------------------------------------------------------------------===//

44/// Class for arbitrary precision integers.
45///
46/// APInt is a functional replacement for common case unsigned integer type like
47/// "unsigned", "unsigned long" or "uint64_t", but also allows non-byte-width
48/// integer sizes and large integer value types such as 3-bits, 15-bits, or more
49/// than 64-bits of precision. APInt provides a variety of arithmetic operators
50/// and methods to manipulate integer values of any bit-width. It supports both
51/// the typical integer arithmetic and comparison operations as well as bitwise
52/// manipulation.
53///
54/// The class has several invariants worth noting:
55///   * All bit, byte, and word positions are zero-based.
56///   * Once the bit width is set, it doesn't change except by the Truncate,
57///     SignExtend, or ZeroExtend operations.
58///   * All binary operators must be on APInt instances of the same bit width.
59///     Attempting to use these operators on instances with different bit
60///     widths will yield an assertion.
61///   * The value is stored canonically as an unsigned value. For operations
62///     where it makes a difference, there are both signed and unsigned variants
63///     of the operation. For example, sdiv and udiv. However, because the bit
64///     widths must be the same, operations such as Mul and Add produce the same
65///     results regardless of whether the values are interpreted as signed or
66///     not.
67///   * In general, the class tries to follow the style of computation that LLVM
68///     uses in its IR. This simplifies its use for LLVM.
69///   * APInt supports zero-bit-width values, but operations that require bits
70///     are not defined on it (e.g. you cannot ask for the sign of a zero-bit
71///     integer).  This means that operations like zero extension and logical
72///     shifts are defined, but sign extension and ashr is not.  Zero bit values
73///     compare and hash equal to themselves, and countLeadingZeros returns 0.
74///
75class LLVM_NODISCARD[[clang::warn_unused_result]] APInt {
76public:
typedef uint64_t WordType;

/// This enum is used to hold the constants we needed for APInt.
enum : unsigned {
  /// Byte size of a word.
  APINT_WORD_SIZE = sizeof(WordType),
  /// Bits in a word.
  APINT_BITS_PER_WORD = APINT_WORD_SIZE * CHAR_BIT8
};

enum class Rounding {
  DOWN,
  TOWARD_ZERO,
  UP,
};

static constexpr WordType WORDTYPE_MAX = ~WordType(0);

/// \name Constructors
/// @{

/// Create a new APInt of numBits width, initialized as val.
///
/// If isSigned is true then val is treated as if it were a signed value
/// (i.e. as an int64_t) and the appropriate sign extension to the bit width
/// will be done. Otherwise, no sign extension occurs (high order bits beyond
/// the range of val are zero filled).
///
/// \param numBits the bit width of the constructed APInt
/// \param val the initial value of the APInt
/// \param isSigned how to treat signedness of val
APInt(unsigned numBits, uint64_t val, bool isSigned = false)
    : BitWidth(numBits) {
  if (isSingleWord()) {
    U.VAL = val;
    clearUnusedBits();
  } else {
    initSlowCase(val, isSigned);
  }
}

/// Construct an APInt of numBits width, initialized as bigVal[].
///
/// Note that bigVal.size() can be smaller or larger than the corresponding
/// bit width but any extraneous bits will be dropped.
///
/// \param numBits the bit width of the constructed APInt
/// \param bigVal a sequence of words to form the initial value of the APInt
APInt(unsigned numBits, ArrayRef<uint64_t> bigVal);

/// Equivalent to APInt(numBits, ArrayRef<uint64_t>(bigVal, numWords)), but
/// deprecated because this constructor is prone to ambiguity with the
/// APInt(unsigned, uint64_t, bool) constructor.
///
/// If this overload is ever deleted, care should be taken to prevent calls
/// from being incorrectly captured by the APInt(unsigned, uint64_t, bool)
/// constructor.
APInt(unsigned numBits, unsigned numWords, const uint64_t bigVal[]);

/// Construct an APInt from a string representation.
///
/// This constructor interprets the string \p str in the given radix. The
/// interpretation stops when the first character that is not suitable for the
/// radix is encountered, or the end of the string. Acceptable radix values
/// are 2, 8, 10, 16, and 36. It is an error for the value implied by the
/// string to require more bits than numBits.
///
/// \param numBits the bit width of the constructed APInt
/// \param str the string to be interpreted
/// \param radix the radix to use for the conversion
APInt(unsigned numBits, StringRef str, uint8_t radix);

/// Default constructor that creates an APInt with a 1-bit zero value.
explicit APInt() : BitWidth(1) { U.VAL = 0; }

/// Copy Constructor.
APInt(const APInt &that) : BitWidth(that.BitWidth) {
  if (isSingleWord())
    U.VAL = that.U.VAL;
  else
    initSlowCase(that);
}

/// Move Constructor.
APInt(APInt &&that) : BitWidth(that.BitWidth) {
  memcpy(&U, &that.U, sizeof(U));
  that.BitWidth = 0;
}

/// Destructor.
~APInt() {
  if (needsCleanup())
    delete[] U.pVal;
}

/// @}
/// \name Value Generators
/// @{

/// Get the '0' value for the specified bit-width.
static APInt getZero(unsigned numBits) { return APInt(numBits, 0); }

/// NOTE: This is soft-deprecated.  Please use `getZero()` instead.
static APInt getNullValue(unsigned numBits) { return getZero(numBits); }

/// Return an APInt zero bits wide.
static APInt getZeroWidth() { return getZero(0); }

/// Gets maximum unsigned value of APInt for specific bit width.
static APInt getMaxValue(unsigned numBits) { return getAllOnes(numBits); }

/// Gets maximum signed value of APInt for a specific bit width.
static APInt getSignedMaxValue(unsigned numBits) {
  APInt API = getAllOnes(numBits);
  API.clearBit(numBits - 1);
  return API;
}

/// Gets minimum unsigned value of APInt for a specific bit width.
static APInt getMinValue(unsigned numBits) { return APInt(numBits, 0); }

/// Gets minimum signed value of APInt for a specific bit width.
static APInt getSignedMinValue(unsigned numBits) {
  APInt API(numBits, 0);
  API.setBit(numBits - 1);
  return API;
}

/// Get the SignMask for a specific bit width.
///
/// This is just a wrapper function of getSignedMinValue(), and it helps code
/// readability when we want to get a SignMask.
static APInt getSignMask(unsigned BitWidth) {
  return getSignedMinValue(BitWidth);
}

/// Return an APInt of a specified width with all bits set.
static APInt getAllOnes(unsigned numBits) {
  return APInt(numBits, WORDTYPE_MAX, true);
}

/// NOTE: This is soft-deprecated.  Please use `getAllOnes()` instead.
static APInt getAllOnesValue(unsigned numBits) { return getAllOnes(numBits); }

/// Return an APInt with exactly one bit set in the result.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo) {
  APInt Res(numBits, 0);
  Res.setBit(BitNo);
  return Res;
}

/// Get a value with a block of bits set.
///
/// Constructs an APInt value that has a contiguous range of bits set. The
/// bits from loBit (inclusive) to hiBit (exclusive) will be set. All other
/// bits will be zero. For example, with parameters(32, 0, 16) you would get
/// 0x0000FFFF. Please call getBitsSetWithWrap if \p loBit may be greater than
/// \p hiBit.
///
/// \param numBits the intended bit width of the result
/// \param loBit the index of the lowest bit set.
/// \param hiBit the index of the highest bit set.
///
/// \returns An APInt value with the requested bits set.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit) {
  APInt Res(numBits, 0);
  Res.setBits(loBit, hiBit);
  return Res;
}

/// Wrap version of getBitsSet.
/// If \p hiBit is bigger than \p loBit, this is same with getBitsSet.
/// If \p hiBit is not bigger than \p loBit, the set bits "wrap". For example,
/// with parameters (32, 28, 4), you would get 0xF000000F.
/// If \p hiBit is equal to \p loBit, you would get a result with all bits
/// set.
static APInt getBitsSetWithWrap(unsigned numBits, unsigned loBit,
                                unsigned hiBit) {
  APInt Res(numBits, 0);
  Res.setBitsWithWrap(loBit, hiBit);
  return Res;
}

/// Constructs an APInt value that has a contiguous range of bits set. The
/// bits from loBit (inclusive) to numBits (exclusive) will be set. All other
/// bits will be zero. For example, with parameters(32, 12) you would get
/// 0xFFFFF000.
///
/// \param numBits the intended bit width of the result
/// \param loBit the index of the lowest bit to set.
///
/// \returns An APInt value with the requested bits set.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit) {
  APInt Res(numBits, 0);
  Res.setBitsFrom(loBit);
  return Res;
}

/// Constructs an APInt value that has the top hiBitsSet bits set.
///
/// \param numBits the bitwidth of the result
/// \param hiBitsSet the number of high-order bits set in the result.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet) {
  APInt Res(numBits, 0);
  Res.setHighBits(hiBitsSet);
  return Res;
}

/// Constructs an APInt value that has the bottom loBitsSet bits set.
///
/// \param numBits the bitwidth of the result
/// \param loBitsSet the number of low-order bits set in the result.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet) {
  APInt Res(numBits, 0);
  Res.setLowBits(loBitsSet);
  return Res;
}

/// Return a value containing V broadcasted over NewLen bits.
static APInt getSplat(unsigned NewLen, const APInt &V);

/// @}
/// \name Value Tests
/// @{

/// Determine if this APInt just has one word to store value.
///
/// \returns true if the number of bits <= 64, false otherwise.
bool isSingleWord() const { return BitWidth <= APINT_BITS_PER_WORD; }

/// Determine sign of this APInt.
///
/// This tests the high bit of this APInt to determine if it is set.
///
/// \returns true if this APInt is negative, false otherwise
bool isNegative() const { return (*this)[BitWidth - 1]; }

/// Determine if this APInt Value is non-negative (>= 0)
///
/// This tests the high bit of the APInt to determine if it is unset.
bool isNonNegative() const { return !isNegative(); }

/// Determine if sign bit of this APInt is set.
///
/// This tests the high bit of this APInt to determine if it is set.
///
/// \returns true if this APInt has its sign bit set, false otherwise.
bool isSignBitSet() const { return (*this)[BitWidth - 1]; }

/// Determine if sign bit of this APInt is clear.
///
/// This tests the high bit of this APInt to determine if it is clear.
///
/// \returns true if this APInt has its sign bit clear, false otherwise.
bool isSignBitClear() const { return !isSignBitSet(); }

/// Determine if this APInt Value is positive.
///
/// This tests if the value of this APInt is positive (> 0). Note
/// that 0 is not a positive value.
///
/// \returns true if this APInt is positive.
bool isStrictlyPositive() const { return isNonNegative() && !isZero(); }

/// Determine if this APInt Value is non-positive (<= 0).
///
/// \returns true if this APInt is non-positive.
bool isNonPositive() const { return !isStrictlyPositive(); }

/// Determine if all bits are set.  This is true for zero-width values.
bool isAllOnes() const {
  if (BitWidth == 0)
    return true;
  if (isSingleWord())
    return U.VAL == WORDTYPE_MAX >> (APINT_BITS_PER_WORD - BitWidth);
  return countTrailingOnesSlowCase() == BitWidth;
}

/// NOTE: This is soft-deprecated.  Please use `isAllOnes()` instead.
bool isAllOnesValue() const { return isAllOnes(); }

/// Determine if this value is zero, i.e. all bits are clear.
bool isZero() const {
  if (isSingleWord())
    return U.VAL == 0;
  return countLeadingZerosSlowCase() == BitWidth;
}

/// NOTE: This is soft-deprecated.  Please use `isZero()` instead.
bool isNullValue() const { return isZero(); }

/// Determine if this is a value of 1.
///
/// This checks to see if the value of this APInt is one.
bool isOne() const {
  if (isSingleWord())
    return U.VAL == 1;
  return countLeadingZerosSlowCase() == BitWidth - 1;
}

/// NOTE: This is soft-deprecated.  Please use `isOne()` instead.
bool isOneValue() const { return isOne(); }

/// Determine if this is the largest unsigned value.
///
/// This checks to see if the value of this APInt is the maximum unsigned
/// value for the APInt's bit width.
bool isMaxValue() const { return isAllOnes(); }

/// Determine if this is the largest signed value.
///
/// This checks to see if the value of this APInt is the maximum signed
/// value for the APInt's bit width.
bool isMaxSignedValue() const {
  if (isSingleWord()) {
    assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 392, __extension__ __PRETTY_FUNCTION__));
    return U.VAL == ((WordType(1) << (BitWidth - 1)) - 1);
  }
  return !isNegative() && countTrailingOnesSlowCase() == BitWidth - 1;
}

/// Determine if this is the smallest unsigned value.
///
/// This checks to see if the value of this APInt is the minimum unsigned
/// value for the APInt's bit width.
bool isMinValue() const { return isZero(); }

/// Determine if this is the smallest signed value.
///
/// This checks to see if the value of this APInt is the minimum signed
/// value for the APInt's bit width.
bool isMinSignedValue() const {
  if (isSingleWord()) {
    assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 410, __extension__ __PRETTY_FUNCTION__));
    return U.VAL == (WordType(1) << (BitWidth - 1));
  }
  return isNegative() && countTrailingZerosSlowCase() == BitWidth - 1;
}

/// Check if this APInt has an N-bits unsigned integer value.
bool isIntN(unsigned N) const { return getActiveBits() <= N; }

/// Check if this APInt has an N-bits signed integer value.
bool isSignedIntN(unsigned N) const { return getMinSignedBits() <= N; }

/// Check if this APInt's value is a power of two greater than zero.
///
/// \returns true if the argument APInt value is a power of two > 0.
bool isPowerOf2() const {
  if (isSingleWord()) {
    assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 427, __extension__ __PRETTY_FUNCTION__));
    return isPowerOf2_64(U.VAL);
  }
  return countPopulationSlowCase() == 1;
}

/// Check if this APInt's negated value is a power of two greater than zero.
bool isNegatedPowerOf2() const {
  assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 435, __extension__ __PRETTY_FUNCTION__));
  if (isNonNegative())
    return false;
  // NegatedPowerOf2 - shifted mask in the top bits.
  unsigned LO = countLeadingOnes();
  unsigned TZ = countTrailingZeros();
  return (LO + TZ) == BitWidth;
}

/// Check if the APInt's value is returned by getSignMask.
///
/// \returns true if this is the value returned by getSignMask.
bool isSignMask() const { return isMinSignedValue(); }

/// Convert APInt to a boolean value.
///
/// This converts the APInt to a boolean value as a test against zero.
bool getBoolValue() const { return !isZero(); }

/// If this value is smaller than the specified limit, return it, otherwise
/// return the limit value.  This causes the value to saturate to the limit.
uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) const {
  return ugt(Limit) ? Limit : getZExtValue();
}

/// Check if the APInt consists of a repeated bit pattern.
///
/// e.g. 0x01010101 satisfies isSplat(8).
/// \param SplatSizeInBits The size of the pattern in bits. Must divide bit
/// width without remainder.
bool isSplat(unsigned SplatSizeInBits) const;

/// \returns true if this APInt value is a sequence of \param numBits ones
/// starting at the least significant bit with the remainder zero.
bool isMask(unsigned numBits) const {
  assert(numBits != 0 && "numBits must be non-zero")(static_cast <bool> (numBits != 0 && "numBits must be non-zero"
) ? void (0) : __assert_fail ("numBits != 0 && \"numBits must be non-zero\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 470, __extension__ __PRETTY_FUNCTION__));
  assert(numBits <= BitWidth && "numBits out of range")(static_cast <bool> (numBits <= BitWidth && "numBits out of range"
) ? void (0) : __assert_fail ("numBits <= BitWidth && \"numBits out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 471, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    return U.VAL == (WORDTYPE_MAX >> (APINT_BITS_PER_WORD - numBits));
  unsigned Ones = countTrailingOnesSlowCase();
  return (numBits == Ones) &&
         ((Ones + countLeadingZerosSlowCase()) == BitWidth);
}

/// \returns true if this APInt is a non-empty sequence of ones starting at
/// the least significant bit with the remainder zero.
/// Ex. isMask(0x0000FFFFU) == true.
bool isMask() const {
  if (isSingleWord())
    return isMask_64(U.VAL);
  unsigned Ones = countTrailingOnesSlowCase();
  return (Ones > 0) && ((Ones + countLeadingZerosSlowCase()) == BitWidth);
}

/// Return true if this APInt value contains a sequence of ones with
/// the remainder zero.
bool isShiftedMask() const {
  if (isSingleWord())
    return isShiftedMask_64(U.VAL);
  unsigned Ones = countPopulationSlowCase();
  unsigned LeadZ = countLeadingZerosSlowCase();
  return (Ones + LeadZ + countTrailingZeros()) == BitWidth;
}

/// Compute an APInt containing numBits highbits from this APInt.
///
/// Get an APInt with the same BitWidth as this APInt, just zero mask the low
/// bits and right shift to the least significant bit.
///
/// \returns the high "numBits" bits of this APInt.
APInt getHiBits(unsigned numBits) const;

/// Compute an APInt containing numBits lowbits from this APInt.
///
/// Get an APInt with the same BitWidth as this APInt, just zero mask the high
/// bits.
///
/// \returns the low "numBits" bits of this APInt.
APInt getLoBits(unsigned numBits) const;

/// Determine if two APInts have the same value, after zero-extending
/// one of them (if needed!) to ensure that the bit-widths match.
static bool isSameValue(const APInt &I1, const APInt &I2) {
  if (I1.getBitWidth() == I2.getBitWidth())
    return I1 == I2;

  if (I1.getBitWidth() > I2.getBitWidth())
    return I1 == I2.zext(I1.getBitWidth());

  return I1.zext(I2.getBitWidth()) == I2;
}

/// Overload to compute a hash_code for an APInt value.
friend hash_code hash_value(const APInt &Arg);

/// This function returns a pointer to the internal storage of the APInt.
/// This is useful for writing out the APInt in binary form without any
/// conversions.
const uint64_t *getRawData() const {
  if (isSingleWord())
    return &U.VAL;
  return &U.pVal[0];
}

/// @}
/// \name Unary Operators
/// @{

/// Postfix increment operator.  Increment *this by 1.
///
/// \returns a new APInt value representing the original value of *this.
APInt operator++(int) {
  APInt API(*this);
  ++(*this);
  return API;
}

/// Prefix increment operator.
///
/// \returns *this incremented by one
APInt &operator++();

/// Postfix decrement operator. Decrement *this by 1.
///
/// \returns a new APInt value representing the original value of *this.
APInt operator--(int) {
  APInt API(*this);
  --(*this);
  return API;
}

/// Prefix decrement operator.
///
/// \returns *this decremented by one.
APInt &operator--();

/// Logical negation operation on this APInt returns true if zero, like normal
/// integers.
bool operator!() const { return isZero(); }

/// @}
/// \name Assignment Operators
/// @{

/// Copy assignment operator.
///
/// \returns *this after assignment of RHS.
APInt &operator=(const APInt &RHS) {
  // The common case (both source or dest being inline) doesn't require
  // allocation or deallocation.
  if (isSingleWord() && RHS.isSingleWord()) {
    U.VAL = RHS.U.VAL;
    BitWidth = RHS.BitWidth;
    return *this;
  }

  assignSlowCase(RHS);
  return *this;
}

/// Move assignment operator.
APInt &operator=(APInt &&that) {
597#ifdef EXPENSIVE_CHECKS
  // Some std::shuffle implementations still do self-assignment.
  if (this == &that)
    return *this;
601#endif
  assert(this != &that && "Self-move not supported")(static_cast <bool> (this != &that && "Self-move not supported"
) ? void (0) : __assert_fail ("this != &that && \"Self-move not supported\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 602, __extension__ __PRETTY_FUNCTION__));
  if (!isSingleWord())
    delete[] U.pVal;

  // Use memcpy so that type based alias analysis sees both VAL and pVal
  // as modified.
  memcpy(&U, &that.U, sizeof(U));

  BitWidth = that.BitWidth;
  that.BitWidth = 0;
  return *this;
}

/// Assignment operator.
///
/// The RHS value is assigned to *this. If the significant bits in RHS exceed
/// the bit width, the excess bits are truncated. If the bit width is larger
/// than 64, the value is zero filled in the unspecified high order bits.
///
/// \returns *this after assignment of RHS value.
APInt &operator=(uint64_t RHS) {
  if (isSingleWord()) {
    U.VAL = RHS;
    return clearUnusedBits();
  }
  U.pVal[0] = RHS;
  memset(U.pVal + 1, 0, (getNumWords() - 1) * APINT_WORD_SIZE);
  return *this;
}

/// Bitwise AND assignment operator.
///
/// Performs a bitwise AND operation on this APInt and RHS. The result is
/// assigned to *this.
///
/// \returns *this after ANDing with RHS.
APInt &operator&=(const APInt &RHS) {
  assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
 "Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 639, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    U.VAL &= RHS.U.VAL;
  else
    andAssignSlowCase(RHS);
  return *this;
}

/// Bitwise AND assignment operator.
///
/// Performs a bitwise AND operation on this APInt and RHS. RHS is
/// logically zero-extended or truncated to match the bit-width of
/// the LHS.
APInt &operator&=(uint64_t RHS) {
  if (isSingleWord()) {
    U.VAL &= RHS;
    return *this;
  }
  U.pVal[0] &= RHS;
  memset(U.pVal + 1, 0, (getNumWords() - 1) * APINT_WORD_SIZE);
  return *this;
}

/// Bitwise OR assignment operator.
///
/// Performs a bitwise OR operation on this APInt and RHS. The result is
/// assigned *this;
///
/// \returns *this after ORing with RHS.
APInt &operator|=(const APInt &RHS) {
  assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
 "Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 669, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    U.VAL |= RHS.U.VAL;
  else
    orAssignSlowCase(RHS);
  return *this;
}

/// Bitwise OR assignment operator.
///
/// Performs a bitwise OR operation on this APInt and RHS. RHS is
/// logically zero-extended or truncated to match the bit-width of
/// the LHS.
APInt &operator|=(uint64_t RHS) {
  if (isSingleWord()) {
    U.VAL |= RHS;
    return clearUnusedBits();
  }
  U.pVal[0] |= RHS;
  return *this;
}

/// Bitwise XOR assignment operator.
///
/// Performs a bitwise XOR operation on this APInt and RHS. The result is
/// assigned to *this.
///
/// \returns *this after XORing with RHS.
APInt &operator^=(const APInt &RHS) {
  assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
 "Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 698, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    U.VAL ^= RHS.U.VAL;
  else
    xorAssignSlowCase(RHS);
  return *this;
}

/// Bitwise XOR assignment operator.
///
/// Performs a bitwise XOR operation on this APInt and RHS. RHS is
/// logically zero-extended or truncated to match the bit-width of
/// the LHS.
APInt &operator^=(uint64_t RHS) {
  if (isSingleWord()) {
    U.VAL ^= RHS;
    return clearUnusedBits();
  }
  U.pVal[0] ^= RHS;
  return *this;
}

/// Multiplication assignment operator.
///
/// Multiplies this APInt by RHS and assigns the result to *this.
///
/// \returns *this
APInt &operator*=(const APInt &RHS);
APInt &operator*=(uint64_t RHS);

/// Addition assignment operator.
///
/// Adds RHS to *this and assigns the result to *this.
///
/// \returns *this
APInt &operator+=(const APInt &RHS);
APInt &operator+=(uint64_t RHS);

/// Subtraction assignment operator.
///
/// Subtracts RHS from *this and assigns the result to *this.
///
/// \returns *this
APInt &operator-=(const APInt &RHS);
APInt &operator-=(uint64_t RHS);

/// Left-shift assignment function.
///
/// Shifts *this left by shiftAmt and assigns the result to *this.
///
/// \returns *this after shifting left by ShiftAmt
APInt &operator<<=(unsigned ShiftAmt) {
  assert(ShiftAmt <= BitWidth && "Invalid shift amount")(static_cast <bool> (ShiftAmt <= BitWidth &&
 "Invalid shift amount") ? void (0) : __assert_fail ("ShiftAmt <= BitWidth && \"Invalid shift amount\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 750, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord()) {
    if (ShiftAmt == BitWidth)
      U.VAL = 0;
    else
      U.VAL <<= ShiftAmt;
    return clearUnusedBits();
  }
  shlSlowCase(ShiftAmt);
  return *this;
}

/// Left-shift assignment function.
///
/// Shifts *this left by shiftAmt and assigns the result to *this.
///
/// \returns *this after shifting left by ShiftAmt
APInt &operator<<=(const APInt &ShiftAmt);

/// @}
/// \name Binary Operators
/// @{

/// Multiplication operator.
///
/// Multiplies this APInt by RHS and returns the result.
APInt operator*(const APInt &RHS) const;

/// Left logical shift operator.
///
/// Shifts this APInt left by \p Bits and returns the result.
APInt operator<<(unsigned Bits) const { return shl(Bits); }

/// Left logical shift operator.
///
/// Shifts this APInt left by \p Bits and returns the result.
APInt operator<<(const APInt &Bits) const { return shl(Bits); }

/// Arithmetic right-shift function.
///
/// Arithmetic right-shift this APInt by shiftAmt.
APInt ashr(unsigned ShiftAmt) const {
  APInt R(*this);
  R.ashrInPlace(ShiftAmt);
  return R;
}

/// Arithmetic right-shift this APInt by ShiftAmt in place.
void ashrInPlace(unsigned ShiftAmt) {
  assert(ShiftAmt <= BitWidth && "Invalid shift amount")(static_cast <bool> (ShiftAmt <= BitWidth &&
 "Invalid shift amount") ? void (0) : __assert_fail ("ShiftAmt <= BitWidth && \"Invalid shift amount\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 799, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord()) {
    int64_t SExtVAL = SignExtend64(U.VAL, BitWidth);
    if (ShiftAmt == BitWidth)
      U.VAL = SExtVAL >> (APINT_BITS_PER_WORD - 1); // Fill with sign bit.
    else
      U.VAL = SExtVAL >> ShiftAmt;
    clearUnusedBits();
    return;
  }
  ashrSlowCase(ShiftAmt);
}

/// Logical right-shift function.
///
/// Logical right-shift this APInt by shiftAmt.
APInt lshr(unsigned shiftAmt) const {
  APInt R(*this);
  R.lshrInPlace(shiftAmt);
  return R;
}

/// Logical right-shift this APInt by ShiftAmt in place.
void lshrInPlace(unsigned ShiftAmt) {
  assert(ShiftAmt <= BitWidth && "Invalid shift amount")(static_cast <bool> (ShiftAmt <= BitWidth &&
 "Invalid shift amount") ? void (0) : __assert_fail ("ShiftAmt <= BitWidth && \"Invalid shift amount\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 823, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord()) {
    if (ShiftAmt == BitWidth)
      U.VAL = 0;
    else
      U.VAL >>= ShiftAmt;
    return;
  }
  lshrSlowCase(ShiftAmt);
}

/// Left-shift function.
///
/// Left-shift this APInt by shiftAmt.
APInt shl(unsigned shiftAmt) const {
  APInt R(*this);
  R <<= shiftAmt;
  return R;
}

/// Rotate left by rotateAmt.
APInt rotl(unsigned rotateAmt) const;

/// Rotate right by rotateAmt.
APInt rotr(unsigned rotateAmt) const;

/// Arithmetic right-shift function.
///
/// Arithmetic right-shift this APInt by shiftAmt.
APInt ashr(const APInt &ShiftAmt) const {
  APInt R(*this);
  R.ashrInPlace(ShiftAmt);
  return R;
}

/// Arithmetic right-shift this APInt by shiftAmt in place.
void ashrInPlace(const APInt &shiftAmt);

/// Logical right-shift function.
///
/// Logical right-shift this APInt by shiftAmt.
APInt lshr(const APInt &ShiftAmt) const {
  APInt R(*this);
  R.lshrInPlace(ShiftAmt);
  return R;
}

/// Logical right-shift this APInt by ShiftAmt in place.
void lshrInPlace(const APInt &ShiftAmt);

/// Left-shift function.
///
/// Left-shift this APInt by shiftAmt.
APInt shl(const APInt &ShiftAmt) const {
  APInt R(*this);
  R <<= ShiftAmt;
  return R;
}

/// Rotate left by rotateAmt.
APInt rotl(const APInt &rotateAmt) const;

/// Rotate right by rotateAmt.
APInt rotr(const APInt &rotateAmt) const;

/// Concatenate the bits from "NewLSB" onto the bottom of *this.  This is
/// equivalent to:
///   (this->zext(NewWidth) << NewLSB.getBitWidth()) | NewLSB.zext(NewWidth)
APInt concat(const APInt &NewLSB) const {
  /// If the result will be small, then both the merged values are small.
  unsigned NewWidth = getBitWidth() + NewLSB.getBitWidth();
  if (NewWidth <= APINT_BITS_PER_WORD)
    return APInt(NewWidth, (U.VAL << NewLSB.getBitWidth()) | NewLSB.U.VAL);
  return concatSlowCase(NewLSB);
}

/// Unsigned division operation.
///
/// Perform an unsigned divide operation on this APInt by RHS. Both this and
/// RHS are treated as unsigned quantities for purposes of this division.
///
/// \returns a new APInt value containing the division result, rounded towards
/// zero.
APInt udiv(const APInt &RHS) const;
APInt udiv(uint64_t RHS) const;

/// Signed division function for APInt.
///
/// Signed divide this APInt by APInt RHS.
///
/// The result is rounded towards zero.
APInt sdiv(const APInt &RHS) const;
APInt sdiv(int64_t RHS) const;

/// Unsigned remainder operation.
///
/// Perform an unsigned remainder operation on this APInt with RHS being the
/// divisor. Both this and RHS are treated as unsigned quantities for purposes
/// of this operation. Note that this is a true remainder operation and not a
/// modulo operation because the sign follows the sign of the dividend which
/// is *this.
///
/// \returns a new APInt value containing the remainder result
APInt urem(const APInt &RHS) const;
uint64_t urem(uint64_t RHS) const;

/// Function for signed remainder operation.
///
/// Signed remainder operation on APInt.
APInt srem(const APInt &RHS) const;
int64_t srem(int64_t RHS) const;

/// Dual division/remainder interface.
///
/// Sometimes it is convenient to divide two APInt values and obtain both the
/// quotient and remainder. This function does both operations in the same
/// computation making it a little more efficient. The pair of input arguments
/// may overlap with the pair of output arguments. It is safe to call
/// udivrem(X, Y, X, Y), for example.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient,
                    APInt &Remainder);
static void udivrem(const APInt &LHS, uint64_t RHS, APInt &Quotient,
                    uint64_t &Remainder);

static void sdivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient,
                    APInt &Remainder);
static void sdivrem(const APInt &LHS, int64_t RHS, APInt &Quotient,
                    int64_t &Remainder);

// Operations that return overflow indicators.
APInt sadd_ov(const APInt &RHS, bool &Overflow) const;
APInt uadd_ov(const APInt &RHS, bool &Overflow) const;
APInt ssub_ov(const APInt &RHS, bool &Overflow) const;
APInt usub_ov(const APInt &RHS, bool &Overflow) const;
APInt sdiv_ov(const APInt &RHS, bool &Overflow) const;
APInt smul_ov(const APInt &RHS, bool &Overflow) const;
APInt umul_ov(const APInt &RHS, bool &Overflow) const;
APInt sshl_ov(const APInt &Amt, bool &Overflow) const;
APInt ushl_ov(const APInt &Amt, bool &Overflow) const;

// Operations that saturate
APInt sadd_sat(const APInt &RHS) const;
APInt uadd_sat(const APInt &RHS) const;
APInt ssub_sat(const APInt &RHS) const;
APInt usub_sat(const APInt &RHS) const;
APInt smul_sat(const APInt &RHS) const;
APInt umul_sat(const APInt &RHS) const;
APInt sshl_sat(const APInt &RHS) const;
APInt ushl_sat(const APInt &RHS) const;

/// Array-indexing support.
///
/// \returns the bit value at bitPosition
bool operator[](unsigned bitPosition) const {
  assert(bitPosition < getBitWidth() && "Bit position out of bounds!")(static_cast <bool> (bitPosition < getBitWidth() &&
 "Bit position out of bounds!") ? void (0) : __assert_fail ("bitPosition < getBitWidth() && \"Bit position out of bounds!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 977, __extension__ __PRETTY_FUNCTION__));
48
←
'?' condition is true→
  return (maskBit(bitPosition) & getWord(bitPosition)) != 0;
49
←
Assuming the condition is true→
50
←
Returning the value 1, which participates in a condition later→
}

/// @}
/// \name Comparison Operators
/// @{

/// Equality operator.
///
/// Compares this APInt with RHS for the validity of the equality
/// relationship.
bool operator==(const APInt &RHS) const {
  assert(BitWidth == RHS.BitWidth && "Comparison requires equal bit widths")(static_cast <bool> (BitWidth == RHS.BitWidth &&
 "Comparison requires equal bit widths") ? void (0) : __assert_fail
 ("BitWidth == RHS.BitWidth && \"Comparison requires equal bit widths\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 990, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    return U.VAL == RHS.U.VAL;
  return equalSlowCase(RHS);
}

/// Equality operator.
///
/// Compares this APInt with a uint64_t for the validity of the equality
/// relationship.
///
/// \returns true if *this == Val
bool operator==(uint64_t Val) const {
  return (isSingleWord() || getActiveBits() <= 64) && getZExtValue() == Val;
11
←
Assuming the condition is false→
12
←
Returning zero, which participates in a condition later→
15
←
Assuming the condition is false→
16
←
Returning zero, which participates in a condition later→
}

/// Equality comparison.
///
/// Compares this APInt with RHS for the validity of the equality
/// relationship.
///
/// \returns true if *this == Val
bool eq(const APInt &RHS) const { return (*this) == RHS; }

/// Inequality operator.
///
/// Compares this APInt with RHS for the validity of the inequality
/// relationship.
///
/// \returns true if *this != Val
bool operator!=(const APInt &RHS) const { return !((*this) == RHS); }

/// Inequality operator.
///
/// Compares this APInt with a uint64_t for the validity of the inequality
/// relationship.
///
/// \returns true if *this != Val
bool operator!=(uint64_t Val) const { return !((*this) == Val); }

/// Inequality comparison
///
/// Compares this APInt with RHS for the validity of the inequality
/// relationship.
///
/// \returns true if *this != Val
bool ne(const APInt &RHS) const { return !((*this) == RHS); }

/// Unsigned less than comparison
///
/// Regards both *this and RHS as unsigned quantities and compares them for
/// the validity of the less-than relationship.
///
/// \returns true if *this < RHS when both are considered unsigned.
bool ult(const APInt &RHS) const { return compare(RHS) < 0; }

/// Unsigned less than comparison
///
/// Regards both *this as an unsigned quantity and compares it with RHS for
/// the validity of the less-than relationship.
///
/// \returns true if *this < RHS when considered unsigned.
bool ult(uint64_t RHS) const {
  // Only need to check active bits if not a single word.
  return (isSingleWord() || getActiveBits() <= 64) && getZExtValue() < RHS;
}

/// Signed less than comparison
///
/// Regards both *this and RHS as signed quantities and compares them for
/// validity of the less-than relationship.
///
/// \returns true if *this < RHS when both are considered signed.
bool slt(const APInt &RHS) const { return compareSigned(RHS) < 0; }

/// Signed less than comparison
///
/// Regards both *this as a signed quantity and compares it with RHS for
/// the validity of the less-than relationship.
///
/// \returns true if *this < RHS when considered signed.
bool slt(int64_t RHS) const {
  return (!isSingleWord() && getMinSignedBits() > 64) ? isNegative()
                                                      : getSExtValue() < RHS;
}

/// Unsigned less or equal comparison
///
/// Regards both *this and RHS as unsigned quantities and compares them for
/// validity of the less-or-equal relationship.
///
/// \returns true if *this <= RHS when both are considered unsigned.
bool ule(const APInt &RHS) const { return compare(RHS) <= 0; }

/// Unsigned less or equal comparison
///
/// Regards both *this as an unsigned quantity and compares it with RHS for
/// the validity of the less-or-equal relationship.
///
/// \returns true if *this <= RHS when considered unsigned.
bool ule(uint64_t RHS) const { return !ugt(RHS); }

/// Signed less or equal comparison
///
/// Regards both *this and RHS as signed quantities and compares them for
/// validity of the less-or-equal relationship.
///
/// \returns true if *this <= RHS when both are considered signed.
bool sle(const APInt &RHS) const { return compareSigned(RHS) <= 0; }

/// Signed less or equal comparison
///
/// Regards both *this as a signed quantity and compares it with RHS for the
/// validity of the less-or-equal relationship.
///
/// \returns true if *this <= RHS when considered signed.
bool sle(uint64_t RHS) const { return !sgt(RHS); }

/// Unsigned greater than comparison
///
/// Regards both *this and RHS as unsigned quantities and compares them for
/// the validity of the greater-than relationship.
///
/// \returns true if *this > RHS when both are considered unsigned.
bool ugt(const APInt &RHS) const { return !ule(RHS); }

/// Unsigned greater than comparison
///
/// Regards both *this as an unsigned quantity and compares it with RHS for
/// the validity of the greater-than relationship.
///
/// \returns true if *this > RHS when considered unsigned.
bool ugt(uint64_t RHS) const {
  // Only need to check active bits if not a single word.
  return (!isSingleWord() && getActiveBits() > 64) || getZExtValue() > RHS;
}

/// Signed greater than comparison
///
/// Regards both *this and RHS as signed quantities and compares them for the
/// validity of the greater-than relationship.
///
/// \returns true if *this > RHS when both are considered signed.
bool sgt(const APInt &RHS) const { return !sle(RHS); }

/// Signed greater than comparison
///
/// Regards both *this as a signed quantity and compares it with RHS for
/// the validity of the greater-than relationship.
///
/// \returns true if *this > RHS when considered signed.
bool sgt(int64_t RHS) const {
  return (!isSingleWord() && getMinSignedBits() > 64) ? !isNegative()
                                                      : getSExtValue() > RHS;
}

/// Unsigned greater or equal comparison
///
/// Regards both *this and RHS as unsigned quantities and compares them for
/// validity of the greater-or-equal relationship.
///
/// \returns true if *this >= RHS when both are considered unsigned.
bool uge(const APInt &RHS) const { return !ult(RHS); }

/// Unsigned greater or equal comparison
///
/// Regards both *this as an unsigned quantity and compares it with RHS for
/// the validity of the greater-or-equal relationship.
///
/// \returns true if *this >= RHS when considered unsigned.
bool uge(uint64_t RHS) const { return !ult(RHS); }

/// Signed greater or equal comparison
///
/// Regards both *this and RHS as signed quantities and compares them for
/// validity of the greater-or-equal relationship.
///
/// \returns true if *this >= RHS when both are considered signed.
bool sge(const APInt &RHS) const { return !slt(RHS); }

/// Signed greater or equal comparison
///
/// Regards both *this as a signed quantity and compares it with RHS for
/// the validity of the greater-or-equal relationship.
///
/// \returns true if *this >= RHS when considered signed.
bool sge(int64_t RHS) const { return !slt(RHS); }

/// This operation tests if there are any pairs of corresponding bits
/// between this APInt and RHS that are both set.
bool intersects(const APInt &RHS) const {
  assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
 "Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1181, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    return (U.VAL & RHS.U.VAL) != 0;
  return intersectsSlowCase(RHS);
}

/// This operation checks that all bits set in this APInt are also set in RHS.
bool isSubsetOf(const APInt &RHS) const {
  assert(BitWidth == RHS.BitWidth && "Bit widths must be the same")(static_cast <bool> (BitWidth == RHS.BitWidth &&
 "Bit widths must be the same") ? void (0) : __assert_fail ("BitWidth == RHS.BitWidth && \"Bit widths must be the same\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1189, __extension__ __PRETTY_FUNCTION__));
  if (isSingleWord())
    return (U.VAL & ~RHS.U.VAL) == 0;
  return isSubsetOfSlowCase(RHS);
}

/// @}
/// \name Resizing Operators
/// @{

/// Truncate to new width.
///
/// Truncate the APInt to a specified width. It is an error to specify a width
/// that is greater than or equal to the current width.
APInt trunc(unsigned width) const;

/// Truncate to new width with unsigned saturation.
///
/// If the APInt, treated as unsigned integer, can be losslessly truncated to
/// the new bitwidth, then return truncated APInt. Else, return max value.
APInt truncUSat(unsigned width) const;

/// Truncate to new width with signed saturation.
///
/// If this APInt, treated as signed integer, can be losslessly truncated to
/// the new bitwidth, then return truncated APInt. Else, return either
/// signed min value if the APInt was negative, or signed max value.
APInt truncSSat(unsigned width) const;

/// Sign extend to a new width.
///
/// This operation sign extends the APInt to a new width. If the high order
/// bit is set, the fill on the left will be done with 1 bits, otherwise zero.
/// It is an error to specify a width that is less than or equal to the
/// current width.
APInt sext(unsigned width) const;

/// Zero extend to a new width.
///
/// This operation zero extends the APInt to a new width. The high order bits
/// are filled with 0 bits.  It is an error to specify a width that is less
/// than or equal to the current width.
APInt zext(unsigned width) const;

/// Sign extend or truncate to width
///
/// Make this APInt have the bit width given by \p width. The value is sign
/// extended, truncated, or left alone to make it that width.
APInt sextOrTrunc(unsigned width) const;

/// Zero extend or truncate to width
///
/// Make this APInt have the bit width given by \p width. The value is zero
/// extended, truncated, or left alone to make it that width.
APInt zextOrTrunc(unsigned width) const;

/// Truncate to width
///
/// Make this APInt have the bit width given by \p width. The value is
/// truncated or left alone to make it that width.
APInt truncOrSelf(unsigned width) const;

/// Sign extend or truncate to width
///
/// Make this APInt have the bit width given by \p width. The value is sign
/// extended, or left alone to make it that width.
APInt sextOrSelf(unsigned width) const;

/// Zero extend or truncate to width
///
/// Make this APInt have the bit width given by \p width. The value is zero
/// extended, or left alone to make it that width.
APInt zextOrSelf(unsigned width) const;

/// @}
/// \name Bit Manipulation Operators
/// @{

/// Set every bit to 1.
void setAllBits() {
  if (isSingleWord())
    U.VAL = WORDTYPE_MAX;
  else
    // Set all the bits in all the words.
    memset(U.pVal, -1, getNumWords() * APINT_WORD_SIZE);
  // Clear the unused ones
  clearUnusedBits();
}

/// Set the given bit to 1 whose position is given as "bitPosition".
void setBit(unsigned BitPosition) {
  assert(BitPosition < BitWidth && "BitPosition out of range")(static_cast <bool> (BitPosition < BitWidth &&
 "BitPosition out of range") ? void (0) : __assert_fail ("BitPosition < BitWidth && \"BitPosition out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1280, __extension__ __PRETTY_FUNCTION__));
  WordType Mask = maskBit(BitPosition);
  if (isSingleWord())
    U.VAL |= Mask;
  else
    U.pVal[whichWord(BitPosition)] |= Mask;
}

/// Set the sign bit to 1.
void setSignBit() { setBit(BitWidth - 1); }

/// Set a given bit to a given value.
void setBitVal(unsigned BitPosition, bool BitValue) {
  if (BitValue)
    setBit(BitPosition);
  else
    clearBit(BitPosition);
}

/// Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
/// This function handles "wrap" case when \p loBit >= \p hiBit, and calls
/// setBits when \p loBit < \p hiBit.
/// For \p loBit == \p hiBit wrap case, set every bit to 1.
void setBitsWithWrap(unsigned loBit, unsigned hiBit) {
  assert(hiBit <= BitWidth && "hiBit out of range")(static_cast <bool> (hiBit <= BitWidth && "hiBit out of range"
) ? void (0) : __assert_fail ("hiBit <= BitWidth && \"hiBit out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1304, __extension__ __PRETTY_FUNCTION__));
  assert(loBit <= BitWidth && "loBit out of range")(static_cast <bool> (loBit <= BitWidth && "loBit out of range"
) ? void (0) : __assert_fail ("loBit <= BitWidth && \"loBit out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1305, __extension__ __PRETTY_FUNCTION__));
  if (loBit < hiBit) {
    setBits(loBit, hiBit);
    return;
  }
  setLowBits(hiBit);
  setHighBits(BitWidth - loBit);
}

/// Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
/// This function handles case when \p loBit <= \p hiBit.
void setBits(unsigned loBit, unsigned hiBit) {
  assert(hiBit <= BitWidth && "hiBit out of range")(static_cast <bool> (hiBit <= BitWidth && "hiBit out of range"
) ? void (0) : __assert_fail ("hiBit <= BitWidth && \"hiBit out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1317, __extension__ __PRETTY_FUNCTION__));
  assert(loBit <= BitWidth && "loBit out of range")(static_cast <bool> (loBit <= BitWidth && "loBit out of range"
) ? void (0) : __assert_fail ("loBit <= BitWidth && \"loBit out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1318, __extension__ __PRETTY_FUNCTION__));
  assert(loBit <= hiBit && "loBit greater than hiBit")(static_cast <bool> (loBit <= hiBit && "loBit greater than hiBit"
) ? void (0) : __assert_fail ("loBit <= hiBit && \"loBit greater than hiBit\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1319, __extension__ __PRETTY_FUNCTION__));
  if (loBit == hiBit)
    return;
  if (loBit < APINT_BITS_PER_WORD && hiBit <= APINT_BITS_PER_WORD) {
    uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - (hiBit - loBit));
    mask <<= loBit;
    if (isSingleWord())
      U.VAL |= mask;
    else
      U.pVal[0] |= mask;
  } else {
    setBitsSlowCase(loBit, hiBit);
  }
}

/// Set the top bits starting from loBit.
void setBitsFrom(unsigned loBit) { return setBits(loBit, BitWidth); }

/// Set the bottom loBits bits.
void setLowBits(unsigned loBits) { return setBits(0, loBits); }

/// Set the top hiBits bits.
void setHighBits(unsigned hiBits) {
  return setBits(BitWidth - hiBits, BitWidth);
}

/// Set every bit to 0.
void clearAllBits() {
  if (isSingleWord())
    U.VAL = 0;
  else
    memset(U.pVal, 0, getNumWords() * APINT_WORD_SIZE);
}

/// Set a given bit to 0.
///
/// Set the given bit to 0 whose position is given as "bitPosition".
void clearBit(unsigned BitPosition) {
  assert(BitPosition < BitWidth && "BitPosition out of range")(static_cast <bool> (BitPosition < BitWidth &&
 "BitPosition out of range") ? void (0) : __assert_fail ("BitPosition < BitWidth && \"BitPosition out of range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1357, __extension__ __PRETTY_FUNCTION__));
  WordType Mask = ~maskBit(BitPosition);
  if (isSingleWord())
    U.VAL &= Mask;
  else
    U.pVal[whichWord(BitPosition)] &= Mask;
}

/// Set bottom loBits bits to 0.
void clearLowBits(unsigned loBits) {
  assert(loBits <= BitWidth && "More bits than bitwidth")(static_cast <bool> (loBits <= BitWidth && "More bits than bitwidth"
) ? void (0) : __assert_fail ("loBits <= BitWidth && \"More bits than bitwidth\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1367, __extension__ __PRETTY_FUNCTION__));
  APInt Keep = getHighBitsSet(BitWidth, BitWidth - loBits);
  *this &= Keep;
}

/// Set the sign bit to 0.
void clearSignBit() { clearBit(BitWidth - 1); }

/// Toggle every bit to its opposite value.
void flipAllBits() {
  if (isSingleWord()) {
    U.VAL ^= WORDTYPE_MAX;
    clearUnusedBits();
  } else {
    flipAllBitsSlowCase();
  }
}

/// Toggles a given bit to its opposite value.
///
/// Toggle a given bit to its opposite value whose position is given
/// as "bitPosition".
void flipBit(unsigned bitPosition);

/// Negate this APInt in place.
void negate() {
  flipAllBits();
  ++(*this);
}

/// Insert the bits from a smaller APInt starting at bitPosition.
void insertBits(const APInt &SubBits, unsigned bitPosition);
void insertBits(uint64_t SubBits, unsigned bitPosition, unsigned numBits);

/// Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
APInt extractBits(unsigned numBits, unsigned bitPosition) const;
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const;

/// @}
/// \name Value Characterization Functions
/// @{

/// Return the number of bits in the APInt.
unsigned getBitWidth() const { return BitWidth; }

/// Get the number of words.
///
/// Here one word's bitwidth equals to that of uint64_t.
///
/// \returns the number of words to hold the integer value of this APInt.
unsigned getNumWords() const { return getNumWords(BitWidth); }

/// Get the number of words.
///
/// *NOTE* Here one word's bitwidth equals to that of uint64_t.
///
/// \returns the number of words to hold the integer value with a given bit
/// width.
static unsigned getNumWords(unsigned BitWidth) {
  return ((uint64_t)BitWidth + APINT_BITS_PER_WORD - 1) / APINT_BITS_PER_WORD;
}

/// Compute the number of active bits in the value
///
/// This function returns the number of active bits which is defined as the
/// bit width minus the number of leading zeros. This is used in several
/// computations to see how "wide" the value is.
unsigned getActiveBits() const { return BitWidth - countLeadingZeros(); }

/// Compute the number of active words in the value of this APInt.
///
/// This is used in conjunction with getActiveData to extract the raw value of
/// the APInt.
unsigned getActiveWords() const {
  unsigned numActiveBits = getActiveBits();
  return numActiveBits ? whichWord(numActiveBits - 1) + 1 : 1;
}

/// Get the minimum bit size for this signed APInt
///
/// Computes the minimum bit width for this APInt while considering it to be a
/// signed (and probably negative) value. If the value is not negative, this
/// function returns the same value as getActiveBits()+1. Otherwise, it
/// returns the smallest bit width that will retain the negative value. For
/// example, -1 can be written as 0b1 or 0xFFFFFFFFFF. 0b1 is shorter and so
/// for -1, this function will always return 1.
unsigned getMinSignedBits() const { return BitWidth - getNumSignBits() + 1; }

/// Get zero extended value
///
/// This method attempts to return the value of this APInt as a zero extended
/// uint64_t. The bitwidth must be <= 64 or the value must fit within a
/// uint64_t. Otherwise an assertion will result.
uint64_t getZExtValue() const {
  if (isSingleWord()) {
    assert(BitWidth && "zero width values not allowed")(static_cast <bool> (BitWidth && "zero width values not allowed"
) ? void (0) : __assert_fail ("BitWidth && \"zero width values not allowed\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1462, __extension__ __PRETTY_FUNCTION__));
    return U.VAL;
  }
  assert(getActiveBits() <= 64 && "Too many bits for uint64_t")(static_cast <bool> (getActiveBits() <= 64 &&
 "Too many bits for uint64_t") ? void (0) : __assert_fail ("getActiveBits() <= 64 && \"Too many bits for uint64_t\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1465, __extension__ __PRETTY_FUNCTION__));
  return U.pVal[0];
}

/// Get sign extended value
///
/// This method attempts to return the value of this APInt as a sign extended
/// int64_t. The bit width must be <= 64 or the value must fit within an
/// int64_t. Otherwise an assertion will result.
int64_t getSExtValue() const {
  if (isSingleWord())
    return SignExtend64(U.VAL, BitWidth);
  assert(getMinSignedBits() <= 64 && "Too many bits for int64_t")(static_cast <bool> (getMinSignedBits() <= 64 &&
 "Too many bits for int64_t") ? void (0) : __assert_fail ("getMinSignedBits() <= 64 && \"Too many bits for int64_t\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/ADT/APInt.h"
, 1477, __extension__ __PRETTY_FUNCTION__));
  return int64_t(U.pVal[0]);
}

/// Get bits required for string value.
///
/// This method determines how many bits are required to hold the APInt
/// equivalent of the string given by \p str.
static unsigned getBitsNeeded(StringRef str, uint8_t radix);

/// The APInt version of the countLeadingZeros functions in
///   MathExtras.h.
///
/// It counts the number of zeros from the most significant bit to the first
/// one bit.
///
/// \returns BitWidth if the value is zero, otherwise returns the number of
///   zeros from the most significant bit to the first one bits.
unsigned countLeadingZeros() const {
  if (isSingleWord()) {
    unsigned unusedBits = APINT_BITS_PER_WORD - BitWidth;
    return llvm::countLeadingZeros(U.VAL) - unusedBits;
  }
  return countLeadingZerosSlowCase();
}

/// Count the number of leading one bits.
///
/// This function is an APInt version of the countLeadingOnes
/// functions in MathExtras.h. It counts the number of ones from the most
/// significant bit to the first zero bit.
///
/// \returns 0 if the high order bit is not set, otherwise returns the number
/// of 1 bits from the most significant to the least
unsigned countLeadingOnes() const {
  if (isSingleWord()) {
    if (LLVM_UNLIKELY(BitWidth == 0)__builtin_expect((bool)(BitWidth == 0), false))
      return 0;
    return llvm::countLeadingOnes(U.VAL << (APINT_BITS_PER_WORD - BitWidth));
  }
  return countLeadingOnesSlowCase();
}

/// Computes the number of leading bits of this APInt that are equal to its
/// sign bit.
unsigned getNumSignBits() const {
  return isNegative() ? countLeadingOnes() : countLeadingZeros();
}

/// Count the number of trailing zero bits.
///
/// This function is an APInt version of the countTrailingZeros
/// functions in MathExtras.h. It counts the number of zeros from the least
/// significant bit to the first set bit.
///
/// \returns BitWidth if the value is zero, otherwise returns the number of
/// zeros from the least significant bit to the first one bit.
unsigned countTrailingZeros() const {
  if (isSingleWord()) {
    unsigned TrailingZeros = llvm::countTrailingZeros(U.VAL);
    return (TrailingZeros > BitWidth ? BitWidth : TrailingZeros);
  }
  return countTrailingZerosSlowCase();
}

/// Count the number of trailing one bits.
///
/// This function is an APInt version of the countTrailingOnes
/// functions in MathExtras.h. It counts the number of ones from the least
/// significant bit to the first zero bit.
///
/// \returns BitWidth if the value is all ones, otherwise returns the number
/// of ones from the least significant bit to the first zero bit.
unsigned countTrailingOnes() const {
  if (isSingleWord())
    return llvm::countTrailingOnes(U.VAL);
  return countTrailingOnesSlowCase();
}

/// Count the number of bits set.
///
/// This function is an APInt version of the countPopulation functions
/// in MathExtras.h. It counts the number of 1 bits in the APInt value.
///
/// \returns 0 if the value is zero, otherwise returns the number of set bits.
unsigned countPopulation() const {
  if (isSingleWord())
    return llvm::countPopulation(U.VAL);
  return countPopulationSlowCase();
}

/// @}
/// \name Conversion Functions
/// @{
void print(raw_ostream &OS, bool isSigned) const;

/// Converts an APInt to a string and append it to Str.  Str is commonly a
/// SmallString.
void toString(SmallVectorImpl<char> &Str, unsigned Radix, bool Signed,
              bool formatAsCLiteral = false) const;

/// Considers the APInt to be unsigned and converts it into a string in the
/// radix given. The radix can be 2, 8, 10 16, or 36.
void toStringUnsigned(SmallVectorImpl<char> &Str, unsigned Radix = 10) const {
  toString(Str, Radix, false, false);
}

/// Considers the APInt to be signed and converts it into a string in the
/// radix given. The radix can be 2, 8, 10, 16, or 36.
void toStringSigned(SmallVectorImpl<char> &Str, unsigned Radix = 10) const {
  toString(Str, Radix, true, false);
}

/// \returns a byte-swapped representation of this APInt Value.
APInt byteSwap() const;

/// \returns the value with the bit representation reversed of this APInt
/// Value.
APInt reverseBits() const;

/// Converts this APInt to a double value.
double roundToDouble(bool isSigned) const;

/// Converts this unsigned APInt to a double value.
double roundToDouble() const { return roundToDouble(false); }

/// Converts this signed APInt to a double value.
double signedRoundToDouble() const { return roundToDouble(true); }

/// Converts APInt bits to a double
///
/// The conversion does not do a translation from integer to double, it just
/// re-interprets the bits as a double. Note that it is valid to do this on
/// any bit width. Exactly 64 bits will be translated.
double bitsToDouble() const { return BitsToDouble(getWord(0)); }

/// Converts APInt bits to a float
///
/// The conversion does not do a translation from integer to float, it just
/// re-interprets the bits as a float. Note that it is valid to do this on
/// any bit width. Exactly 32 bits will be translated.
float bitsToFloat() const {
  return BitsToFloat(static_cast<uint32_t>(getWord(0)));
}

/// Converts a double to APInt bits.
///
/// The conversion does not do a translation from double to integer, it just
/// re-interprets the bits of the double.
static APInt doubleToBits(double V) {
  return APInt(sizeof(double) * CHAR_BIT8, DoubleToBits(V));
}

/// Converts a float to APInt bits.
///
/// The conversion does not do a translation from float to integer, it just
/// re-interprets the bits of the float.
static APInt floatToBits(float V) {
  return APInt(sizeof(float) * CHAR_BIT8, FloatToBits(V));
}

/// @}
/// \name Mathematics Operations
/// @{

/// \returns the floor log base 2 of this APInt.
unsigned logBase2() const { return getActiveBits() - 1; }

/// \returns the ceil log base 2 of this APInt.
unsigned ceilLogBase2() const {
  APInt temp(*this);
  --temp;
  return temp.getActiveBits();
}

/// \returns the nearest log base 2 of this APInt. Ties round up.
///
/// NOTE: When we have a BitWidth of 1, we define:
///
///   log2(0) = UINT32_MAX
///   log2(1) = 0
///
/// to get around any mathematical concerns resulting from
/// referencing 2 in a space where 2 does no exist.
unsigned nearestLogBase2() const;

/// \returns the log base 2 of this APInt if its an exact power of two, -1
/// otherwise
int32_t exactLogBase2() const {
  if (!isPowerOf2())
    return -1;
  return logBase2();
}

/// Compute the square root.
APInt sqrt() const;

/// Get the absolute value.  If *this is < 0 then return -(*this), otherwise
/// *this.  Note that the "most negative" signed number (e.g. -128 for 8 bit
/// wide APInt) is unchanged due to how negation works.
APInt abs() const {
  if (isNegative())
    return -(*this);
  return *this;
}

/// \returns the multiplicative inverse for a given modulo.
APInt multiplicativeInverse(const APInt &modulo) const;

/// @}
/// \name Building-block Operations for APInt and APFloat
/// @{

// These building block operations operate on a representation of arbitrary
// precision, two's-complement, bignum integer values. They should be
// sufficient to implement APInt and APFloat bignum requirements. Inputs are
// generally a pointer to the base of an array of integer parts, representing
// an unsigned bignum, and a count of how many parts there are.

/// Sets the least significant part of a bignum to the input value, and zeroes
/// out higher parts.
static void tcSet(WordType *, WordType, unsigned);

/// Assign one bignum to another.
static void tcAssign(WordType *, const WordType *, unsigned);

/// Returns true if a bignum is zero, false otherwise.
static bool tcIsZero(const WordType *, unsigned);

/// Extract the given bit of a bignum; returns 0 or 1.  Zero-based.
static int tcExtractBit(const WordType *, unsigned bit);

/// Copy the bit vector of width srcBITS from SRC, starting at bit srcLSB, to
/// DST, of dstCOUNT parts, such that the bit srcLSB becomes the least
/// significant bit of DST.  All high bits above srcBITS in DST are
/// zero-filled.
static void tcExtract(WordType *, unsigned dstCount, const WordType *,
                      unsigned srcBits, unsigned srcLSB);

/// Set the given bit of a bignum.  Zero-based.
static void tcSetBit(WordType *, unsigned bit);

/// Clear the given bit of a bignum.  Zero-based.
static void tcClearBit(WordType *, unsigned bit);

/// Returns the bit number of the least or most significant set bit of a
/// number.  If the input number has no bits set -1U is returned.
static unsigned tcLSB(const WordType *, unsigned n);
static unsigned tcMSB(const WordType *parts, unsigned n);

/// Negate a bignum in-place.
static void tcNegate(WordType *, unsigned);

/// DST += RHS + CARRY where CARRY is zero or one.  Returns the carry flag.
static WordType tcAdd(WordType *, const WordType *, WordType carry, unsigned);
/// DST += RHS.  Returns the carry flag.
static WordType tcAddPart(WordType *, WordType, unsigned);

/// DST -= RHS + CARRY where CARRY is zero or one. Returns the carry flag.
static WordType tcSubtract(WordType *, const WordType *, WordType carry,
                           unsigned);
/// DST -= RHS.  Returns the carry flag.
static WordType tcSubtractPart(WordType *, WordType, unsigned);

/// DST += SRC * MULTIPLIER + PART   if add is true
/// DST  = SRC * MULTIPLIER + PART   if add is false
///
/// Requires 0 <= DSTPARTS <= SRCPARTS + 1.  If DST overlaps SRC they must
/// start at the same point, i.e. DST == SRC.
///
/// If DSTPARTS == SRC_PARTS + 1 no overflow occurs and zero is returned.
/// Otherwise DST is filled with the least significant DSTPARTS parts of the
/// result, and if all of the omitted higher parts were zero return zero,
/// otherwise overflow occurred and return one.
static int tcMultiplyPart(WordType *dst, const WordType *src,
                          WordType multiplier, WordType carry,
                          unsigned srcParts, unsigned dstParts, bool add);

/// DST = LHS * RHS, where DST has the same width as the operands and is
/// filled with the least significant parts of the result.  Returns one if
/// overflow occurred, otherwise zero.  DST must be disjoint from both
/// operands.
static int tcMultiply(WordType *, const WordType *, const WordType *,
                      unsigned);

/// DST = LHS * RHS, where DST has width the sum of the widths of the
/// operands. No overflow occurs. DST must be disjoint from both operands.
static void tcFullMultiply(WordType *, const WordType *, const WordType *,
                           unsigned, unsigned);

/// If RHS is zero LHS and REMAINDER are left unchanged, return one.
/// Otherwise set LHS to LHS / RHS with the fractional part discarded, set
/// REMAINDER to the remainder, return zero.  i.e.
///
///  OLD_LHS = RHS * LHS + REMAINDER
///
/// SCRATCH is a bignum of the same size as the operands and result for use by
/// the routine; its contents need not be initialized and are destroyed.  LHS,
/// REMAINDER and SCRATCH must be distinct.
static int tcDivide(WordType *lhs, const WordType *rhs, WordType *remainder,
                    WordType *scratch, unsigned parts);

/// Shift a bignum left Count bits. Shifted in bits are zero. There are no
/// restrictions on Count.
static void tcShiftLeft(WordType *, unsigned Words, unsigned Count);

/// Shift a bignum right Count bits.  Shifted in bits are zero.  There are no
/// restrictions on Count.
static void tcShiftRight(WordType *, unsigned Words, unsigned Count);

/// Comparison (unsigned) of two bignums.
static int tcCompare(const WordType *, const WordType *, unsigned);

/// Increment a bignum in-place.  Return the carry flag.
static WordType tcIncrement(WordType *dst, unsigned parts) {
  return tcAddPart(dst, 1, parts);
}

/// Decrement a bignum in-place.  Return the borrow flag.
static WordType tcDecrement(WordType *dst, unsigned parts) {
  return tcSubtractPart(dst, 1, parts);
}

/// Used to insert APInt objects, or objects that contain APInt objects, into
///  FoldingSets.
void Profile(FoldingSetNodeID &id) const;

/// debug method
void dump() const;

/// Returns whether this instance allocated memory.
bool needsCleanup() const { return !isSingleWord(); }

1810private:
/// This union is used to store the integer value. When the
/// integer bit-width <= 64, it uses VAL, otherwise it uses pVal.
union {
  uint64_t VAL;   ///< Used to store the <= 64 bits integer value.
  uint64_t *pVal; ///< Used to store the >64 bits integer value.
} U;

unsigned BitWidth; ///< The number of bits in this APInt.

friend struct DenseMapInfo<APInt>;
friend class APSInt;

/// This constructor is used only internally for speed of construction of
/// temporaries. It is unsafe since it takes ownership of the pointer, so it
/// is not public.
APInt(uint64_t *val, unsigned bits) : BitWidth(bits) { U.pVal = val; }

/// Determine which word a bit is in.
///
/// \returns the word position for the specified bit position.
static unsigned whichWord(unsigned bitPosition) {
  return bitPosition / APINT_BITS_PER_WORD;
}

/// Determine which bit in a word the specified bit position is in.
static unsigned whichBit(unsigned bitPosition) {
  return bitPosition % APINT_BITS_PER_WORD;
}

/// Get a single bit mask.
///
/// \returns a uint64_t with only bit at "whichBit(bitPosition)" set
/// This method generates and returns a uint64_t (word) mask for a single
/// bit at a specific bit position. This is used to mask the bit in the
/// corresponding word.
static uint64_t maskBit(unsigned bitPosition) {
  return 1ULL << whichBit(bitPosition);
}

/// Clear unused high order bits
///
/// This method is used internally to clear the top "N" bits in the high order
/// word that are not used by the APInt. This is needed after the most
/// significant word is assigned a value to ensure that those bits are
/// zero'd out.
APInt &clearUnusedBits() {
  // Compute how many bits are used in the final word.
  unsigned WordBits = ((BitWidth - 1) % APINT_BITS_PER_WORD) + 1;

  // Mask out the high bits.
  uint64_t mask = WORDTYPE_MAX >> (APINT_BITS_PER_WORD - WordBits);
  if (LLVM_UNLIKELY(BitWidth == 0)__builtin_expect((bool)(BitWidth == 0), false))
    mask = 0;

  if (isSingleWord())
    U.VAL &= mask;
  else
    U.pVal[getNumWords() - 1] &= mask;
  return *this;
}

/// Get the word corresponding to a bit position
/// \returns the corresponding word for the specified bit position.
uint64_t getWord(unsigned bitPosition) const {
  return isSingleWord() ? U.VAL : U.pVal[whichWord(bitPosition)];
}

/// Utility method to change the bit width of this APInt to new bit width,
/// allocating and/or deallocating as necessary. There is no guarantee on the
/// value of any bits upon return. Caller should populate the bits after.
void reallocate(unsigned NewBitWidth);

/// Convert a char array into an APInt
///
/// \param radix 2, 8, 10, 16, or 36
/// Converts a string into a number.  The string must be non-empty
/// and well-formed as a number of the given base. The bit-width
/// must be sufficient to hold the result.
///
/// This is used by the constructors that take string arguments.
///
/// StringRef::getAsInteger is superficially similar but (1) does
/// not assume that the string is well-formed and (2) grows the
/// result to hold the input.
void fromString(unsigned numBits, StringRef str, uint8_t radix);

/// An internal division function for dividing APInts.
///
/// This is used by the toString method to divide by the radix. It simply
/// provides a more convenient form of divide for internal use since KnuthDiv
/// has specific constraints on its inputs. If those constraints are not met
/// then it provides a simpler form of divide.
static void divide(const WordType *LHS, unsigned lhsWords,
                   const WordType *RHS, unsigned rhsWords, WordType *Quotient,
                   WordType *Remainder);

/// out-of-line slow case for inline constructor
void initSlowCase(uint64_t val, bool isSigned);

/// shared code between two array constructors
void initFromArray(ArrayRef<uint64_t> array);

/// out-of-line slow case for inline copy constructor
void initSlowCase(const APInt &that);

/// out-of-line slow case for shl
void shlSlowCase(unsigned ShiftAmt);

/// out-of-line slow case for lshr.
void lshrSlowCase(unsigned ShiftAmt);

/// out-of-line slow case for ashr.
void ashrSlowCase(unsigned ShiftAmt);

/// out-of-line slow case for operator=
void assignSlowCase(const APInt &RHS);

/// out-of-line slow case for operator==
bool equalSlowCase(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for countLeadingZeros
unsigned countLeadingZerosSlowCase() const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for countLeadingOnes.
unsigned countLeadingOnesSlowCase() const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for countTrailingZeros.
unsigned countTrailingZerosSlowCase() const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for countTrailingOnes
unsigned countTrailingOnesSlowCase() const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for countPopulation
unsigned countPopulationSlowCase() const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for intersects.
bool intersectsSlowCase(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for isSubsetOf.
bool isSubsetOfSlowCase(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));

/// out-of-line slow case for setBits.
void setBitsSlowCase(unsigned loBit, unsigned hiBit);

/// out-of-line slow case for flipAllBits.
void flipAllBitsSlowCase();

/// out-of-line slow case for concat.
APInt concatSlowCase(const APInt &NewLSB) const;

/// out-of-line slow case for operator&=.
void andAssignSlowCase(const APInt &RHS);

/// out-of-line slow case for operator|=.
void orAssignSlowCase(const APInt &RHS);

/// out-of-line slow case for operator^=.
void xorAssignSlowCase(const APInt &RHS);

/// Unsigned comparison. Returns -1, 0, or 1 if this APInt is less than, equal
/// to, or greater than RHS.
int compare(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));

/// Signed comparison. Returns -1, 0, or 1 if this APInt is less than, equal
/// to, or greater than RHS.
int compareSigned(const APInt &RHS) const LLVM_READONLY__attribute__((__pure__));

/// @}
1979};

1981inline bool operator==(uint64_t V1, const APInt &V2) { return V2 == V1; }

1983inline bool operator!=(uint64_t V1, const APInt &V2) { return V2 != V1; }

1985/// Unary bitwise complement operator.
1986///
1987/// \returns an APInt that is the bitwise complement of \p v.
1988inline APInt operator~(APInt v) {
v.flipAllBits();
return v;
1991}

1993inline APInt operator&(APInt a, const APInt &b) {
a &= b;
return a;
1996}

1998inline APInt operator&(const APInt &a, APInt &&b) {
b &= a;
return std::move(b);
2001}

2003inline APInt operator&(APInt a, uint64_t RHS) {
a &= RHS;
return a;
2006}

2008inline APInt operator&(uint64_t LHS, APInt b) {
b &= LHS;
return b;
2011}

2013inline APInt operator|(APInt a, const APInt &b) {
a |= b;
return a;
2016}

2018inline APInt operator|(const APInt &a, APInt &&b) {
b |= a;
return std::move(b);
2021}

2023inline APInt operator|(APInt a, uint64_t RHS) {
a |= RHS;
return a;
2026}

2028inline APInt operator|(uint64_t LHS, APInt b) {
b |= LHS;
return b;
2031}

2033inline APInt operator^(APInt a, const APInt &b) {
a ^= b;
return a;
2036}

2038inline APInt operator^(const APInt &a, APInt &&b) {
b ^= a;
return std::move(b);
2041}

2043inline APInt operator^(APInt a, uint64_t RHS) {
a ^= RHS;
return a;
2046}

2048inline APInt operator^(uint64_t LHS, APInt b) {
b ^= LHS;
return b;
2051}

2053inline raw_ostream &operator<<(raw_ostream &OS, const APInt &I) {
I.print(OS, true);
return OS;
2056}

2058inline APInt operator-(APInt v) {
v.negate();
return v;
2061}

2063inline APInt operator+(APInt a, const APInt &b) {
a += b;
return a;
2066}

2068inline APInt operator+(const APInt &a, APInt &&b) {
b += a;
return std::move(b);
2071}

2073inline APInt operator+(APInt a, uint64_t RHS) {
a += RHS;
return a;
2076}

2078inline APInt operator+(uint64_t LHS, APInt b) {
b += LHS;
return b;
2081}

2083inline APInt operator-(APInt a, const APInt &b) {
a -= b;
return a;
2086}

2088inline APInt operator-(const APInt &a, APInt &&b) {
b.negate();
b += a;
return std::move(b);
2092}

2094inline APInt operator-(APInt a, uint64_t RHS) {
a -= RHS;
return a;
2097}

2099inline APInt operator-(uint64_t LHS, APInt b) {
b.negate();
b += LHS;
return b;
2103}

2105inline APInt operator*(APInt a, uint64_t RHS) {
a *= RHS;
return a;
2108}

2110inline APInt operator*(uint64_t LHS, APInt b) {
b *= LHS;
return b;
2113}

2115namespace APIntOps {

2117/// Determine the smaller of two APInts considered to be signed.
2118inline const APInt &smin(const APInt &A, const APInt &B) {
return A.slt(B) ? A : B;
2120}

2122/// Determine the larger of two APInts considered to be signed.
2123inline const APInt &smax(const APInt &A, const APInt &B) {
return A.sgt(B) ? A : B;
2125}

2127/// Determine the smaller of two APInts considered to be unsigned.
2128inline const APInt &umin(const APInt &A, const APInt &B) {
return A.ult(B) ? A : B;
2130}

2132/// Determine the larger of two APInts considered to be unsigned.
2133inline const APInt &umax(const APInt &A, const APInt &B) {
return A.ugt(B) ? A : B;
2135}

2137/// Compute GCD of two unsigned APInt values.
2138///
2139/// This function returns the greatest common divisor of the two APInt values
2140/// using Stein's algorithm.
2141///
2142/// \returns the greatest common divisor of A and B.
2143APInt GreatestCommonDivisor(APInt A, APInt B);

2145/// Converts the given APInt to a double value.
2146///
2147/// Treats the APInt as an unsigned value for conversion purposes.
2148inline double RoundAPIntToDouble(const APInt &APIVal) {
return APIVal.roundToDouble();
2150}

2152/// Converts the given APInt to a double value.
2153///
2154/// Treats the APInt as a signed value for conversion purposes.
2155inline double RoundSignedAPIntToDouble(const APInt &APIVal) {
return APIVal.signedRoundToDouble();
2157}

2159/// Converts the given APInt to a float value.
2160inline float RoundAPIntToFloat(const APInt &APIVal) {
return float(RoundAPIntToDouble(APIVal));
2162}

2164/// Converts the given APInt to a float value.
2165///
2166/// Treats the APInt as a signed value for conversion purposes.
2167inline float RoundSignedAPIntToFloat(const APInt &APIVal) {
return float(APIVal.signedRoundToDouble());
2169}

2171/// Converts the given double value into a APInt.
2172///
2173/// This function convert a double value to an APInt value.
2174APInt RoundDoubleToAPInt(double Double, unsigned width);

2176/// Converts a float value into a APInt.
2177///
2178/// Converts a float value into an APInt value.
2179inline APInt RoundFloatToAPInt(float Float, unsigned width) {
return RoundDoubleToAPInt(double(Float), width);
2181}

2183/// Return A unsign-divided by B, rounded by the given rounding mode.
2184APInt RoundingUDiv(const APInt &A, const APInt &B, APInt::Rounding RM);

2186/// Return A sign-divided by B, rounded by the given rounding mode.
2187APInt RoundingSDiv(const APInt &A, const APInt &B, APInt::Rounding RM);

2189/// Let q(n) = An^2 + Bn + C, and BW = bit width of the value range
2190/// (e.g. 32 for i32).
2191/// This function finds the smallest number n, such that
2192/// (a) n >= 0 and q(n) = 0, or
2193/// (b) n >= 1 and q(n-1) and q(n), when evaluated in the set of all
2194///     integers, belong to two different intervals [Rk, Rk+R),
2195///     where R = 2^BW, and k is an integer.
2196/// The idea here is to find when q(n) "overflows" 2^BW, while at the
2197/// same time "allowing" subtraction. In unsigned modulo arithmetic a
2198/// subtraction (treated as addition of negated numbers) would always
2199/// count as an overflow, but here we want to allow values to decrease
2200/// and increase as long as they are within the same interval.
2201/// Specifically, adding of two negative numbers should not cause an
2202/// overflow (as long as the magnitude does not exceed the bit width).
2203/// On the other hand, given a positive number, adding a negative
2204/// number to it can give a negative result, which would cause the
2205/// value to go from [-2^BW, 0) to [0, 2^BW). In that sense, zero is
2206/// treated as a special case of an overflow.
2207///
2208/// This function returns None if after finding k that minimizes the
2209/// positive solution to q(n) = kR, both solutions are contained between
2210/// two consecutive integers.
2211///
2212/// There are cases where q(n) > T, and q(n+1) < T (assuming evaluation
2213/// in arithmetic modulo 2^BW, and treating the values as signed) by the
2214/// virtue of *signed* overflow. This function will *not* find such an n,
2215/// however it may find a value of n satisfying the inequalities due to
2216/// an *unsigned* overflow (if the values are treated as unsigned).
2217/// To find a solution for a signed overflow, treat it as a problem of
2218/// finding an unsigned overflow with a range with of BW-1.
2219///
2220/// The returned value may have a different bit width from the input
2221/// coefficients.
2222Optional<APInt> SolveQuadraticEquationWrap(APInt A, APInt B, APInt C,
                                         unsigned RangeWidth);

2225/// Compare two values, and if they are different, return the position of the
2226/// most significant bit that is different in the values.
2227Optional<unsigned> GetMostSignificantDifferentBit(const APInt &A,
                                                const APInt &B);

2230/// Splat/Merge neighboring bits to widen/narrow the bitmask represented
2231/// by \param A to \param NewBitWidth bits.
2232///
2233/// e.g. ScaleBitMask(0b0101, 8) -> 0b00110011
2234/// e.g. ScaleBitMask(0b00011011, 4) -> 0b0111
2235/// A.getBitwidth() or NewBitWidth must be a whole multiples of the other.
2236///
2237/// TODO: Do we need a mode where all bits must be set when merging down?
2238APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth);
2239} // namespace APIntOps

2241// See friend declaration above. This additional declaration is required in
2242// order to compile LLVM with IBM xlC compiler.
2243hash_code hash_value(const APInt &Arg);

2245/// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst
2246/// with the integer held in IntVal.
2247void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, unsigned StoreBytes);

2249/// LoadIntFromMemory - Loads the integer stored in the LoadBytes bytes starting
2250/// from Src into IntVal, which is assumed to be wide enough and to hold zero.
2251void LoadIntFromMemory(APInt &IntVal, const uint8_t *Src, unsigned LoadBytes);

2253/// Provide DenseMapInfo for APInt.
2254template <> struct DenseMapInfo<APInt> {
static inline APInt getEmptyKey() {
  APInt V(nullptr, 0);
  V.U.VAL = 0;
  return V;
}

static inline APInt getTombstoneKey() {
  APInt V(nullptr, 0);
  V.U.VAL = 1;
  return V;
}

static unsigned getHashValue(const APInt &Key);

static bool isEqual(const APInt &LHS, const APInt &RHS) {
  return LHS.getBitWidth() == RHS.getBitWidth() && LHS == RHS;
}
2272};

2274} // namespace llvm

2276#endif

←

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h

→

1//===- CodeGen/ValueTypes.h - Low-Level Target independ. types --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the set of low-level target independent types which various
10// values in the code generator are.  This allows the target specific behavior
11// of instructions to be described to target independent passes.
12//
13//===----------------------------------------------------------------------===//

15#ifndef LLVM_CODEGEN_VALUETYPES_H
16#define LLVM_CODEGEN_VALUETYPES_H

18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/MachineValueType.h"
20#include "llvm/Support/MathExtras.h"
21#include "llvm/Support/TypeSize.h"
22#include "llvm/Support/WithColor.h"
23#include <cassert>
24#include <cstdint>
25#include <string>

27namespace llvm {

class LLVMContext;
class Type;

/// Extended Value Type. Capable of holding value types which are not native
/// for any processor (such as the i12345 type), as well as the types an MVT
/// can represent.
struct EVT {
private:
  MVT V = MVT::INVALID_SIMPLE_VALUE_TYPE;
  Type *LLVMTy = nullptr;

public:
  constexpr EVT() = default;
  constexpr EVT(MVT::SimpleValueType SVT) : V(SVT) {}
  constexpr EVT(MVT S) : V(S) {}

  bool operator==(EVT VT) const {
    return !(*this != VT);
21
←
Calling 'EVT::operator!='→
28
←
Returning from 'EVT::operator!='→
29
←
Returning zero, which participates in a condition later→
  }
  bool operator!=(EVT VT) const {
    if (V.SimpleTy != VT.V.SimpleTy)
22
←
Assuming 'V.SimpleTy' is equal to 'VT.V.SimpleTy'→
23
←
Taking false branch→
      return true;
    if (V.SimpleTy == MVT::INVALID_SIMPLE_VALUE_TYPE)
24
←
Assuming field 'SimpleTy' is equal to INVALID_SIMPLE_VALUE_TYPE→
25
←
Taking true branch→
      return LLVMTy != VT.LLVMTy;
26
←
Assuming 'LLVMTy' is not equal to 'VT.LLVMTy'→
27
←
Returning the value 1, which participates in a condition later→
    return false;
  }

  /// Returns the EVT that represents a floating-point type with the given
  /// number of bits. There are two floating-point types with 128 bits - this
  /// returns f128 rather than ppcf128.
  static EVT getFloatingPointVT(unsigned BitWidth) {
    return MVT::getFloatingPointVT(BitWidth);
  }

  /// Returns the EVT that represents an integer with the given number of
  /// bits.
  static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth) {
    MVT M = MVT::getIntegerVT(BitWidth);
    if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
      return M;
    return getExtendedIntegerVT(Context, BitWidth);
  }

  /// Returns the EVT that represents a vector NumElements in length, where
  /// each element is of type VT.
  static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements,
                         bool IsScalable = false) {
    MVT M = MVT::getVectorVT(VT.V, NumElements, IsScalable);
    if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
      return M;
    return getExtendedVectorVT(Context, VT, NumElements, IsScalable);
  }

  /// Returns the EVT that represents a vector EC.Min elements in length,
  /// where each element is of type VT.
  static EVT getVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) {
    MVT M = MVT::getVectorVT(VT.V, EC);
    if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
      return M;
    return getExtendedVectorVT(Context, VT, EC);
  }

  /// Return a vector with the same number of elements as this vector, but
  /// with the element type converted to an integer type with the same
  /// bitwidth.
  EVT changeVectorElementTypeToInteger() const {
    if (isSimple())
      return getSimpleVT().changeVectorElementTypeToInteger();
    return changeExtendedVectorElementTypeToInteger();
  }

  /// Return a VT for a vector type whose attributes match ourselves
  /// with the exception of the element type that is chosen by the caller.
  EVT changeVectorElementType(EVT EltVT) const {
    if (isSimple()) {
      assert(EltVT.isSimple() &&(static_cast <bool> (EltVT.isSimple() && "Can't change simple vector VT to have extended element VT"
) ? void (0) : __assert_fail ("EltVT.isSimple() && \"Can't change simple vector VT to have extended element VT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 105, __extension__ __PRETTY_FUNCTION__))
             "Can't change simple vector VT to have extended element VT")(static_cast <bool> (EltVT.isSimple() && "Can't change simple vector VT to have extended element VT"
) ? void (0) : __assert_fail ("EltVT.isSimple() && \"Can't change simple vector VT to have extended element VT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 105, __extension__ __PRETTY_FUNCTION__));
      return getSimpleVT().changeVectorElementType(EltVT.getSimpleVT());
    }
    return changeExtendedVectorElementType(EltVT);
  }

  /// Return the type converted to an equivalently sized integer or vector
  /// with integer element type. Similar to changeVectorElementTypeToInteger,
  /// but also handles scalars.
  EVT changeTypeToInteger() {
    if (isVector())
      return changeVectorElementTypeToInteger();

    if (isSimple())
      return getSimpleVT().changeTypeToInteger();
    return changeExtendedTypeToInteger();
  }

  /// Test if the given EVT has zero size, this will fail if called on a
  /// scalable type
  bool isZeroSized() const {
    return !isScalableVector() && getSizeInBits() == 0;
  }

  /// Test if the given EVT is simple (as opposed to being extended).
  bool isSimple() const {
    return V.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE;
  }

  /// Test if the given EVT is extended (as opposed to being simple).
  bool isExtended() const {
    return !isSimple();
  }

  /// Return true if this is a FP or a vector FP type.
  bool isFloatingPoint() const {
    return isSimple() ? V.isFloatingPoint() : isExtendedFloatingPoint();
  }

  /// Return true if this is an integer or a vector integer type.
  bool isInteger() const {
    return isSimple() ? V.isInteger() : isExtendedInteger();
  }

  /// Return true if this is an integer, but not a vector.
  bool isScalarInteger() const {
    return isSimple() ? V.isScalarInteger() : isExtendedScalarInteger();
  }

  /// Return true if this is a vector value type.
  bool isVector() const {
    return isSimple() ? V.isVector() : isExtendedVector();
34
←
'?' condition is false→
35
←
Returning value, which participates in a condition later→
  }

  /// Return true if this is a vector type where the runtime
  /// length is machine dependent
  bool isScalableVector() const {
    return isSimple() ? V.isScalableVector() : isExtendedScalableVector();
  }

  bool isFixedLengthVector() const {
    return isSimple() ? V.isFixedLengthVector()
                      : isExtendedFixedLengthVector();
  }

  /// Return true if this is a 16-bit vector type.
  bool is16BitVector() const {
    return isSimple() ? V.is16BitVector() : isExtended16BitVector();
  }

  /// Return true if this is a 32-bit vector type.
  bool is32BitVector() const {
    return isSimple() ? V.is32BitVector() : isExtended32BitVector();
  }

  /// Return true if this is a 64-bit vector type.
  bool is64BitVector() const {
    return isSimple() ? V.is64BitVector() : isExtended64BitVector();
  }

  /// Return true if this is a 128-bit vector type.
  bool is128BitVector() const {
    return isSimple() ? V.is128BitVector() : isExtended128BitVector();
  }

  /// Return true if this is a 256-bit vector type.
  bool is256BitVector() const {
    return isSimple() ? V.is256BitVector() : isExtended256BitVector();
  }

  /// Return true if this is a 512-bit vector type.
  bool is512BitVector() const {
    return isSimple() ? V.is512BitVector() : isExtended512BitVector();
  }

  /// Return true if this is a 1024-bit vector type.
  bool is1024BitVector() const {
    return isSimple() ? V.is1024BitVector() : isExtended1024BitVector();
  }

  /// Return true if this is a 2048-bit vector type.
  bool is2048BitVector() const {
    return isSimple() ? V.is2048BitVector() : isExtended2048BitVector();
  }

  /// Return true if this is an overloaded type for TableGen.
  bool isOverloaded() const {
    return (V==MVT::iAny || V==MVT::fAny || V==MVT::vAny || V==MVT::iPTRAny);
  }

  /// Return true if the bit size is a multiple of 8.
  bool isByteSized() const {
    return !isZeroSized() && getSizeInBits().isKnownMultipleOf(8);
  }

  /// Return true if the size is a power-of-two number of bytes.
  bool isRound() const {
    if (isScalableVector())
      return false;
    unsigned BitSize = getSizeInBits();
    return BitSize >= 8 && !(BitSize & (BitSize - 1));
  }

  /// Return true if this has the same number of bits as VT.
  bool bitsEq(EVT VT) const {
    if (EVT::operator==(VT)) return true;
    return getSizeInBits() == VT.getSizeInBits();
  }

  /// Return true if we know at compile time this has more bits than VT.
  bool knownBitsGT(EVT VT) const {
    return TypeSize::isKnownGT(getSizeInBits(), VT.getSizeInBits());
  }

  /// Return true if we know at compile time this has more than or the same
  /// bits as VT.
  bool knownBitsGE(EVT VT) const {
    return TypeSize::isKnownGE(getSizeInBits(), VT.getSizeInBits());
  }

  /// Return true if we know at compile time this has fewer bits than VT.
  bool knownBitsLT(EVT VT) const {
    return TypeSize::isKnownLT(getSizeInBits(), VT.getSizeInBits());
  }

  /// Return true if we know at compile time this has fewer than or the same
  /// bits as VT.
  bool knownBitsLE(EVT VT) const {
    return TypeSize::isKnownLE(getSizeInBits(), VT.getSizeInBits());
  }

  /// Return true if this has more bits than VT.
  bool bitsGT(EVT VT) const {
    if (EVT::operator==(VT)) return false;
    assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 260, __extension__ __PRETTY_FUNCTION__))
           "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 260, __extension__ __PRETTY_FUNCTION__));
    return knownBitsGT(VT);
  }

  /// Return true if this has no less bits than VT.
  bool bitsGE(EVT VT) const {
    if (EVT::operator==(VT)) return true;
    assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 268, __extension__ __PRETTY_FUNCTION__))
           "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 268, __extension__ __PRETTY_FUNCTION__));
    return knownBitsGE(VT);
  }

  /// Return true if this has less bits than VT.
  bool bitsLT(EVT VT) const {
    if (EVT::operator==(VT)) return false;
    assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 276, __extension__ __PRETTY_FUNCTION__))
           "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 276, __extension__ __PRETTY_FUNCTION__));
    return knownBitsLT(VT);
  }

  /// Return true if this has no more bits than VT.
  bool bitsLE(EVT VT) const {
    if (EVT::operator==(VT)) return true;
    assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 284, __extension__ __PRETTY_FUNCTION__))
           "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
 void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 284, __extension__ __PRETTY_FUNCTION__));
    return knownBitsLE(VT);
  }

  /// Return the SimpleValueType held in the specified simple EVT.
  MVT getSimpleVT() const {
    assert(isSimple() && "Expected a SimpleValueType!")(static_cast <bool> (isSimple() && "Expected a SimpleValueType!"
) ? void (0) : __assert_fail ("isSimple() && \"Expected a SimpleValueType!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 290, __extension__ __PRETTY_FUNCTION__));
    return V;
  }

  /// If this is a vector type, return the element type, otherwise return
  /// this.
  EVT getScalarType() const {
    return isVector() ? getVectorElementType() : *this;
  }

  /// Given a vector type, return the type of each element.
  EVT getVectorElementType() const {
    assert(isVector() && "Invalid vector type!")(static_cast <bool> (isVector() && "Invalid vector type!"
) ? void (0) : __assert_fail ("isVector() && \"Invalid vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 302, __extension__ __PRETTY_FUNCTION__));
    if (isSimple())
      return V.getVectorElementType();
    return getExtendedVectorElementType();
  }

  /// Given a vector type, return the number of elements it contains.
  unsigned getVectorNumElements() const {
    assert(isVector() && "Invalid vector type!")(static_cast <bool> (isVector() && "Invalid vector type!"
) ? void (0) : __assert_fail ("isVector() && \"Invalid vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 310, __extension__ __PRETTY_FUNCTION__));

    if (isScalableVector())
      llvm::reportInvalidSizeRequest(
          "Possible incorrect use of EVT::getVectorNumElements() for "
          "scalable vector. Scalable flag may be dropped, use "
          "EVT::getVectorElementCount() instead");

    return isSimple() ? V.getVectorNumElements()
                      : getExtendedVectorNumElements();
  }

  // Given a (possibly scalable) vector type, return the ElementCount
  ElementCount getVectorElementCount() const {
    assert((isVector()) && "Invalid vector type!")(static_cast <bool> ((isVector()) && "Invalid vector type!"
) ? void (0) : __assert_fail ("(isVector()) && \"Invalid vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 324, __extension__ __PRETTY_FUNCTION__));
    if (isSimple())
      return V.getVectorElementCount();

    return getExtendedVectorElementCount();
  }

  /// Given a vector type, return the minimum number of elements it contains.
  unsigned getVectorMinNumElements() const {
    return getVectorElementCount().getKnownMinValue();
  }

  /// Return the size of the specified value type in bits.
  ///
  /// If the value type is a scalable vector type, the scalable property will
  /// be set and the runtime size will be a positive integer multiple of the
  /// base size.
  TypeSize getSizeInBits() const {
    if (isSimple())
      return V.getSizeInBits();
    return getExtendedSizeInBits();
  }

  /// Return the size of the specified fixed width value type in bits. The
  /// function will assert if the type is scalable.
  uint64_t getFixedSizeInBits() const {
    return getSizeInBits().getFixedSize();
  }

  uint64_t getScalarSizeInBits() const {
    return getScalarType().getSizeInBits().getFixedSize();
  }

  /// Return the number of bytes overwritten by a store of the specified value
  /// type.
  ///
  /// If the value type is a scalable vector type, the scalable property will
  /// be set and the runtime size will be a positive integer multiple of the
  /// base size.
  TypeSize getStoreSize() const {
    TypeSize BaseSize = getSizeInBits();
    return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()};
  }

  /// Return the number of bits overwritten by a store of the specified value
  /// type.
  ///
  /// If the value type is a scalable vector type, the scalable property will
  /// be set and the runtime size will be a positive integer multiple of the
  /// base size.
  TypeSize getStoreSizeInBits() const {
    return getStoreSize() * 8;
  }

  /// Rounds the bit-width of the given integer EVT up to the nearest power of
  /// two (and at least to eight), and returns the integer EVT with that
  /// number of bits.
  EVT getRoundIntegerType(LLVMContext &Context) const {
    assert(isInteger() && !isVector() && "Invalid integer type!")(static_cast <bool> (isInteger() && !isVector()
 && "Invalid integer type!") ? void (0) : __assert_fail
 ("isInteger() && !isVector() && \"Invalid integer type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 382, __extension__ __PRETTY_FUNCTION__));
    unsigned BitWidth = getSizeInBits();
    if (BitWidth <= 8)
      return EVT(MVT::i8);
    return getIntegerVT(Context, 1 << Log2_32_Ceil(BitWidth));
  }

  /// Finds the smallest simple value type that is greater than or equal to
  /// half the width of this EVT. If no simple value type can be found, an
  /// extended integer value type of half the size (rounded up) is returned.
  EVT getHalfSizedIntegerVT(LLVMContext &Context) const {
    assert(isInteger() && !isVector() && "Invalid integer type!")(static_cast <bool> (isInteger() && !isVector()
 && "Invalid integer type!") ? void (0) : __assert_fail
 ("isInteger() && !isVector() && \"Invalid integer type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 393, __extension__ __PRETTY_FUNCTION__));
    unsigned EVTSize = getSizeInBits();
    for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
        IntVT <= MVT::LAST_INTEGER_VALUETYPE; ++IntVT) {
      EVT HalfVT = EVT((MVT::SimpleValueType)IntVT);
      if (HalfVT.getSizeInBits() * 2 >= EVTSize)
        return HalfVT;
    }
    return getIntegerVT(Context, (EVTSize + 1) / 2);
  }

  /// Return a VT for an integer vector type with the size of the
  /// elements doubled. The typed returned may be an extended type.
  EVT widenIntegerVectorElementType(LLVMContext &Context) const {
    EVT EltVT = getVectorElementType();
    EltVT = EVT::getIntegerVT(Context, 2 * EltVT.getSizeInBits());
    return EVT::getVectorVT(Context, EltVT, getVectorElementCount());
  }

  // Return a VT for a vector type with the same element type but
  // half the number of elements. The type returned may be an
  // extended type.
  EVT getHalfNumVectorElementsVT(LLVMContext &Context) const {
    EVT EltVT = getVectorElementType();
    auto EltCnt = getVectorElementCount();
    assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!")(static_cast <bool> (EltCnt.isKnownEven() && "Splitting vector, but not in half!"
) ? void (0) : __assert_fail ("EltCnt.isKnownEven() && \"Splitting vector, but not in half!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 418, __extension__ __PRETTY_FUNCTION__));
    return EVT::getVectorVT(Context, EltVT, EltCnt.divideCoefficientBy(2));
  }

  // Return a VT for a vector type with the same element type but
  // double the number of elements. The type returned may be an
  // extended type.
  EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const {
    EVT EltVT = getVectorElementType();
    auto EltCnt = getVectorElementCount();
    return EVT::getVectorVT(Context, EltVT, EltCnt * 2);
  }

  /// Returns true if the given vector is a power of 2.
  bool isPow2VectorType() const {
    unsigned NElts = getVectorMinNumElements();
    return !(NElts & (NElts - 1));
  }

  /// Widens the length of the given vector EVT up to the nearest power of 2
  /// and returns that type.
  EVT getPow2VectorType(LLVMContext &Context) const {
    if (!isPow2VectorType()) {
      ElementCount NElts = getVectorElementCount();
      unsigned NewMinCount = 1 << Log2_32_Ceil(NElts.getKnownMinValue());
      NElts = ElementCount::get(NewMinCount, NElts.isScalable());
      return EVT::getVectorVT(Context, getVectorElementType(), NElts);
    }
    else {
      return *this;
    }
  }

  /// This function returns value type as a string, e.g. "i32".
  std::string getEVTString() const;

  /// This method returns an LLVM type corresponding to the specified EVT.
  /// For integer types, this returns an unsigned type. Note that this will
  /// abort for types that cannot be represented.
  Type *getTypeForEVT(LLVMContext &Context) const;

  /// Return the value type corresponding to the specified type.
  /// This returns all pointers as iPTR.  If HandleUnknown is true, unknown
  /// types are returned as Other, otherwise they are invalid.
  static EVT getEVT(Type *Ty, bool HandleUnknown = false);

  intptr_t getRawBits() const {
    if (isSimple())
      return V.SimpleTy;
    else
      return (intptr_t)(LLVMTy);
  }

  /// A meaningless but well-behaved order, useful for constructing
  /// containers.
  struct compareRawBits {
    bool operator()(EVT L, EVT R) const {
      if (L.V.SimpleTy == R.V.SimpleTy)
        return L.LLVMTy < R.LLVMTy;
      else
        return L.V.SimpleTy < R.V.SimpleTy;
    }
  };

private:
  // Methods for handling the Extended-type case in functions above.
  // These are all out-of-line to prevent users of this header file
  // from having a dependency on Type.h.
  EVT changeExtendedTypeToInteger() const;
  EVT changeExtendedVectorElementType(EVT EltVT) const;
  EVT changeExtendedVectorElementTypeToInteger() const;
  static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth);
  static EVT getExtendedVectorVT(LLVMContext &C, EVT VT, unsigned NumElements,
                                 bool IsScalable);
  static EVT getExtendedVectorVT(LLVMContext &Context, EVT VT,
                                 ElementCount EC);
  bool isExtendedFloatingPoint() const LLVM_READONLY__attribute__((__pure__));
  bool isExtendedInteger() const LLVM_READONLY__attribute__((__pure__));
  bool isExtendedScalarInteger() const LLVM_READONLY__attribute__((__pure__));
  bool isExtendedVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended16BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended32BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended64BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended128BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended256BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended512BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended1024BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtended2048BitVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtendedFixedLengthVector() const LLVM_READONLY__attribute__((__pure__));
  bool isExtendedScalableVector() const LLVM_READONLY__attribute__((__pure__));
  EVT getExtendedVectorElementType() const;
  unsigned getExtendedVectorNumElements() const LLVM_READONLY__attribute__((__pure__));
  ElementCount getExtendedVectorElementCount() const LLVM_READONLY__attribute__((__pure__));
  TypeSize getExtendedSizeInBits() const LLVM_READONLY__attribute__((__pure__));
};

514} // end namespace llvm

516#endif // LLVM_CODEGEN_VALUETYPES_H

←

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h

1//===- llvm/DataLayout.h - Data size & alignment info -----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines layout properties related to datatype size/offset/alignment
10// information.  It uses lazy annotations to cache information about how
11// structure types are laid out and used.
12//
13// This structure should be created once, filled in if the defaults are not
14// correct and then passed around by const&.  None of the members functions
15// require modification to the object.
16//
17//===----------------------------------------------------------------------===//

19#ifndef LLVM_IR_DATALAYOUT_H
20#define LLVM_IR_DATALAYOUT_H

22#include "llvm/ADT/APInt.h"
23#include "llvm/ADT/ArrayRef.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/StringRef.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Support/Alignment.h"
33#include "llvm/Support/TrailingObjects.h"
34#include "llvm/Support/TypeSize.h"
35#include <cassert>
36#include <cstdint>
37#include <string>

39// This needs to be outside of the namespace, to avoid conflict with llvm-c
40// decl.
41using LLVMTargetDataRef = struct LLVMOpaqueTargetData *;

43namespace llvm {

45class GlobalVariable;
46class LLVMContext;
47class Module;
48class StructLayout;
49class Triple;
50class Value;

52/// Enum used to categorize the alignment types stored by LayoutAlignElem
53enum AlignTypeEnum {
INVALID_ALIGN = 0,
INTEGER_ALIGN = 'i',
VECTOR_ALIGN = 'v',
FLOAT_ALIGN = 'f',
AGGREGATE_ALIGN = 'a'
59};

61// FIXME: Currently the DataLayout string carries a "preferred alignment"
62// for types. As the DataLayout is module/global, this should likely be
63// sunk down to an FTTI element that is queried rather than a global
64// preference.

66/// Layout alignment element.
67///
68/// Stores the alignment data associated with a given alignment type (integer,
69/// vector, float) and type bit width.
70///
71/// \note The unusual order of elements in the structure attempts to reduce
72/// padding and make the structure slightly more cache friendly.
73struct LayoutAlignElem {
/// Alignment type from \c AlignTypeEnum
unsigned AlignType : 8;
unsigned TypeBitWidth : 24;
Align ABIAlign;
Align PrefAlign;

static LayoutAlignElem get(AlignTypeEnum align_type, Align abi_align,
                           Align pref_align, uint32_t bit_width);

bool operator==(const LayoutAlignElem &rhs) const;
84};

86/// Layout pointer alignment element.
87///
88/// Stores the alignment data associated with a given pointer and address space.
89///
90/// \note The unusual order of elements in the structure attempts to reduce
91/// padding and make the structure slightly more cache friendly.
92struct PointerAlignElem {
Align ABIAlign;
Align PrefAlign;
uint32_t TypeByteWidth;
uint32_t AddressSpace;
uint32_t IndexWidth;

/// Initializer
static PointerAlignElem get(uint32_t AddressSpace, Align ABIAlign,
                            Align PrefAlign, uint32_t TypeByteWidth,
                            uint32_t IndexWidth);

bool operator==(const PointerAlignElem &rhs) const;
105};

107/// A parsed version of the target data layout string in and methods for
108/// querying it.
109///
110/// The target data layout string is specified *by the target* - a frontend
111/// generating LLVM IR is required to generate the right target data for the
112/// target being codegen'd to.
113class DataLayout {
114public:
enum class FunctionPtrAlignType {
  /// The function pointer alignment is independent of the function alignment.
  Independent,
  /// The function pointer alignment is a multiple of the function alignment.
  MultipleOfFunctionAlign,
};
121private:
/// Defaults to false.
bool BigEndian;

unsigned AllocaAddrSpace;
MaybeAlign StackNaturalAlign;
unsigned ProgramAddrSpace;
unsigned DefaultGlobalsAddrSpace;

MaybeAlign FunctionPtrAlign;
FunctionPtrAlignType TheFunctionPtrAlignType;

enum ManglingModeT {
  MM_None,
  MM_ELF,
  MM_MachO,
  MM_WinCOFF,
  MM_WinCOFFX86,
  MM_GOFF,
  MM_Mips,
  MM_XCOFF
};
ManglingModeT ManglingMode;

SmallVector<unsigned char, 8> LegalIntWidths;

/// Primitive type alignment data. This is sorted by type and bit
/// width during construction.
using AlignmentsTy = SmallVector<LayoutAlignElem, 16>;
AlignmentsTy Alignments;

AlignmentsTy::const_iterator
findAlignmentLowerBound(AlignTypeEnum AlignType, uint32_t BitWidth) const {
  return const_cast<DataLayout *>(this)->findAlignmentLowerBound(AlignType,
                                                                 BitWidth);
}

AlignmentsTy::iterator
findAlignmentLowerBound(AlignTypeEnum AlignType, uint32_t BitWidth);

/// The string representation used to create this DataLayout
std::string StringRepresentation;

using PointersTy = SmallVector<PointerAlignElem, 8>;
PointersTy Pointers;

const PointerAlignElem &getPointerAlignElem(uint32_t AddressSpace) const;

// The StructType -> StructLayout map.
mutable void *LayoutMap = nullptr;

/// Pointers in these address spaces are non-integral, and don't have a
/// well-defined bitwise representation.
SmallVector<unsigned, 8> NonIntegralAddressSpaces;

/// Attempts to set the alignment of the given type. Returns an error
/// description on failure.
Error setAlignment(AlignTypeEnum align_type, Align abi_align,
                   Align pref_align, uint32_t bit_width);

/// Attempts to set the alignment of a pointer in the given address space.
/// Returns an error description on failure.
Error setPointerAlignment(uint32_t AddrSpace, Align ABIAlign, Align PrefAlign,
                          uint32_t TypeByteWidth, uint32_t IndexWidth);

/// Internal helper to get alignment for integer of given bitwidth.
Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;

/// Internal helper method that returns requested alignment for type.
Align getAlignment(Type *Ty, bool abi_or_pref) const;

/// Attempts to parse a target data specification string and reports an error
/// if the string is malformed.
Error parseSpecifier(StringRef Desc);

// Free all internal data structures.
void clear();

199public:
/// Constructs a DataLayout from a specification string. See reset().
explicit DataLayout(StringRef LayoutDescription) {
  reset(LayoutDescription);
}

/// Initialize target data from properties stored in the module.
explicit DataLayout(const Module *M);

DataLayout(const DataLayout &DL) { *this = DL; }

~DataLayout(); // Not virtual, do not subclass this class

DataLayout &operator=(const DataLayout &DL) {
  clear();
  StringRepresentation = DL.StringRepresentation;
  BigEndian = DL.isBigEndian();
  AllocaAddrSpace = DL.AllocaAddrSpace;
  StackNaturalAlign = DL.StackNaturalAlign;
  FunctionPtrAlign = DL.FunctionPtrAlign;
  TheFunctionPtrAlignType = DL.TheFunctionPtrAlignType;
  ProgramAddrSpace = DL.ProgramAddrSpace;
  DefaultGlobalsAddrSpace = DL.DefaultGlobalsAddrSpace;
  ManglingMode = DL.ManglingMode;
  LegalIntWidths = DL.LegalIntWidths;
  Alignments = DL.Alignments;
  Pointers = DL.Pointers;
  NonIntegralAddressSpaces = DL.NonIntegralAddressSpaces;
  return *this;
}

bool operator==(const DataLayout &Other) const;
bool operator!=(const DataLayout &Other) const { return !(*this == Other); }

void init(const Module *M);

/// Parse a data layout string (with fallback to default values).
void reset(StringRef LayoutDescription);

/// Parse a data layout string and return the layout. Return an error
/// description on failure.
static Expected<DataLayout> parse(StringRef LayoutDescription);

/// Layout endianness...
bool isLittleEndian() const { return !BigEndian; }
40
←
Assuming field 'BigEndian' is false, which participates in a condition later→
41
←
Returning the value 1, which participates in a condition later→
bool isBigEndian() const { return BigEndian; }

/// Returns the string representation of the DataLayout.
///
/// This representation is in the same format accepted by the string
/// constructor above. This should not be used to compare two DataLayout as
/// different string can represent the same layout.
const std::string &getStringRepresentation() const {
  return StringRepresentation;
}

/// Test if the DataLayout was constructed from an empty string.
bool isDefault() const { return StringRepresentation.empty(); }

/// Returns true if the specified type is known to be a native integer
/// type supported by the CPU.
///
/// For example, i64 is not native on most 32-bit CPUs and i37 is not native
/// on any known one. This returns false if the integer width is not legal.
///
/// The width is specified in bits.
bool isLegalInteger(uint64_t Width) const {
  return llvm::is_contained(LegalIntWidths, Width);
}

bool isIllegalInteger(uint64_t Width) const { return !isLegalInteger(Width); }

/// Returns true if the given alignment exceeds the natural stack alignment.
bool exceedsNaturalStackAlignment(Align Alignment) const {
  return StackNaturalAlign && (Alignment > *StackNaturalAlign);
}

Align getStackAlignment() const {
  assert(StackNaturalAlign && "StackNaturalAlign must be defined")(static_cast <bool> (StackNaturalAlign && "StackNaturalAlign must be defined"
) ? void (0) : __assert_fail ("StackNaturalAlign && \"StackNaturalAlign must be defined\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h"
, 277, __extension__ __PRETTY_FUNCTION__));
  return *StackNaturalAlign;
}

unsigned getAllocaAddrSpace() const { return AllocaAddrSpace; }

/// Returns the alignment of function pointers, which may or may not be
/// related to the alignment of functions.
/// \see getFunctionPtrAlignType
MaybeAlign getFunctionPtrAlign() const { return FunctionPtrAlign; }

/// Return the type of function pointer alignment.
/// \see getFunctionPtrAlign
FunctionPtrAlignType getFunctionPtrAlignType() const {
  return TheFunctionPtrAlignType;
}

unsigned getProgramAddressSpace() const { return ProgramAddrSpace; }
unsigned getDefaultGlobalsAddressSpace() const {
  return DefaultGlobalsAddrSpace;
}

bool hasMicrosoftFastStdCallMangling() const {
  return ManglingMode == MM_WinCOFFX86;
}

/// Returns true if symbols with leading question marks should not receive IR
/// mangling. True for Windows mangling modes.
bool doNotMangleLeadingQuestionMark() const {
  return ManglingMode == MM_WinCOFF || ManglingMode == MM_WinCOFFX86;
}

bool hasLinkerPrivateGlobalPrefix() const { return ManglingMode == MM_MachO; }

StringRef getLinkerPrivateGlobalPrefix() const {
  if (ManglingMode == MM_MachO)
    return "l";
  return "";
}

char getGlobalPrefix() const {
  switch (ManglingMode) {
  case MM_None:
  case MM_ELF:
  case MM_GOFF:
  case MM_Mips:
  case MM_WinCOFF:
  case MM_XCOFF:
    return '\0';
  case MM_MachO:
  case MM_WinCOFFX86:
    return '_';
  }
  llvm_unreachable("invalid mangling mode")::llvm::llvm_unreachable_internal("invalid mangling mode", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h"
, 330);
}

StringRef getPrivateGlobalPrefix() const {
  switch (ManglingMode) {
  case MM_None:
    return "";
  case MM_ELF:
  case MM_WinCOFF:
    return ".L";
  case MM_GOFF:
    return "@";
  case MM_Mips:
    return "$";
  case MM_MachO:
  case MM_WinCOFFX86:
    return "L";
  case MM_XCOFF:
    return "L..";
  }
  llvm_unreachable("invalid mangling mode")::llvm::llvm_unreachable_internal("invalid mangling mode", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h"
, 350);
}

static const char *getManglingComponent(const Triple &T);

/// Returns true if the specified type fits in a native integer type
/// supported by the CPU.
///
/// For example, if the CPU only supports i32 as a native integer type, then
/// i27 fits in a legal integer type but i45 does not.
bool fitsInLegalInteger(unsigned Width) const {
  for (unsigned LegalIntWidth : LegalIntWidths)
    if (Width <= LegalIntWidth)
      return true;
  return false;
}

/// Layout pointer alignment
Align getPointerABIAlignment(unsigned AS) const;

/// Return target's alignment for stack-based pointers
/// FIXME: The defaults need to be removed once all of
/// the backends/clients are updated.
Align getPointerPrefAlignment(unsigned AS = 0) const;

/// Layout pointer size
/// FIXME: The defaults need to be removed once all of
/// the backends/clients are updated.
unsigned getPointerSize(unsigned AS = 0) const;

/// Returns the maximum index size over all address spaces.
unsigned getMaxIndexSize() const;

// Index size used for address calculation.
unsigned getIndexSize(unsigned AS) const;

/// Return the address spaces containing non-integral pointers.  Pointers in
/// this address space don't have a well-defined bitwise representation.
ArrayRef<unsigned> getNonIntegralAddressSpaces() const {
  return NonIntegralAddressSpaces;
}

bool isNonIntegralAddressSpace(unsigned AddrSpace) const {
  ArrayRef<unsigned> NonIntegralSpaces = getNonIntegralAddressSpaces();
  return is_contained(NonIntegralSpaces, AddrSpace);
}

bool isNonIntegralPointerType(PointerType *PT) const {
  return isNonIntegralAddressSpace(PT->getAddressSpace());
}

bool isNonIntegralPointerType(Type *Ty) const {
  auto *PTy = dyn_cast<PointerType>(Ty);
  return PTy && isNonIntegralPointerType(PTy);
}

/// Layout pointer size, in bits
/// FIXME: The defaults need to be removed once all of
/// the backends/clients are updated.
unsigned getPointerSizeInBits(unsigned AS = 0) const {
  return getPointerSize(AS) * 8;
}

/// Returns the maximum index size over all address spaces.
unsigned getMaxIndexSizeInBits() const {
  return getMaxIndexSize() * 8;
}

/// Size in bits of index used for address calculation in getelementptr.
unsigned getIndexSizeInBits(unsigned AS) const {
  return getIndexSize(AS) * 8;
}

/// Layout pointer size, in bits, based on the type.  If this function is
/// called with a pointer type, then the type size of the pointer is returned.
/// If this function is called with a vector of pointers, then the type size
/// of the pointer is returned.  This should only be called with a pointer or
/// vector of pointers.
unsigned getPointerTypeSizeInBits(Type *) const;

/// Layout size of the index used in GEP calculation.
/// The function should be called with pointer or vector of pointers type.
unsigned getIndexTypeSizeInBits(Type *Ty) const;

unsigned getPointerTypeSize(Type *Ty) const {
  return getPointerTypeSizeInBits(Ty) / 8;
}

/// Size examples:
///
/// Type        SizeInBits  StoreSizeInBits  AllocSizeInBits[*]
/// ----        ----------  ---------------  ---------------
///  i1            1           8                8
///  i8            8           8                8
///  i19          19          24               32
///  i32          32          32               32
///  i100        100         104              128
///  i128        128         128              128
///  Float        32          32               32
///  Double       64          64               64
///  X86_FP80     80          80               96
///
/// [*] The alloc size depends on the alignment, and thus on the target.
///     These values are for x86-32 linux.

/// Returns the number of bits necessary to hold the specified type.
///
/// If Ty is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// For example, returns 36 for i36 and 80 for x86_fp80. The type passed must
/// have a size (Type::isSized() must return true).
TypeSize getTypeSizeInBits(Type *Ty) const;

/// Returns the maximum number of bytes that may be overwritten by
/// storing the specified type.
///
/// If Ty is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// For example, returns 5 for i36 and 10 for x86_fp80.
TypeSize getTypeStoreSize(Type *Ty) const {
  TypeSize BaseSize = getTypeSizeInBits(Ty);
  return { (BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable() };
}

/// Returns the maximum number of bits that may be overwritten by
/// storing the specified type; always a multiple of 8.
///
/// If Ty is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// For example, returns 40 for i36 and 80 for x86_fp80.
TypeSize getTypeStoreSizeInBits(Type *Ty) const {
  return 8 * getTypeStoreSize(Ty);
}

/// Returns true if no extra padding bits are needed when storing the
/// specified type.
///
/// For example, returns false for i19 that has a 24-bit store size.
bool typeSizeEqualsStoreSize(Type *Ty) const {
  return getTypeSizeInBits(Ty) == getTypeStoreSizeInBits(Ty);
}

/// Returns the offset in bytes between successive objects of the
/// specified type, including alignment padding.
///
/// If Ty is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// This is the amount that alloca reserves for this type. For example,
/// returns 12 or 16 for x86_fp80, depending on alignment.
TypeSize getTypeAllocSize(Type *Ty) const {
  // Round up to the next alignment boundary.
  return alignTo(getTypeStoreSize(Ty), getABITypeAlignment(Ty));
}

/// Returns the offset in bits between successive objects of the
/// specified type, including alignment padding; always a multiple of 8.
///
/// If Ty is a scalable vector type, the scalable property will be set and
/// the runtime size will be a positive integer multiple of the base size.
///
/// This is the amount that alloca reserves for this type. For example,
/// returns 96 or 128 for x86_fp80, depending on alignment.
TypeSize getTypeAllocSizeInBits(Type *Ty) const {
  return 8 * getTypeAllocSize(Ty);
}

/// Returns the minimum ABI-required alignment for the specified type.
/// FIXME: Deprecate this function once migration to Align is over.
uint64_t getABITypeAlignment(Type *Ty) const;

/// Returns the minimum ABI-required alignment for the specified type.
Align getABITypeAlign(Type *Ty) const;

/// Helper function to return `Alignment` if it's set or the result of
/// `getABITypeAlignment(Ty)`, in any case the result is a valid alignment.
inline Align getValueOrABITypeAlignment(MaybeAlign Alignment,
                                        Type *Ty) const {
  return Alignment ? *Alignment : getABITypeAlign(Ty);
}

/// Returns the minimum ABI-required alignment for an integer type of
/// the specified bitwidth.
Align getABIIntegerTypeAlignment(unsigned BitWidth) const {
  return getIntegerAlignment(BitWidth, /* abi_or_pref */ true);
}

/// Returns the preferred stack/global alignment for the specified
/// type.
///
/// This is always at least as good as the ABI alignment.
/// FIXME: Deprecate this function once migration to Align is over.
uint64_t getPrefTypeAlignment(Type *Ty) const;

/// Returns the preferred stack/global alignment for the specified
/// type.
///
/// This is always at least as good as the ABI alignment.
Align getPrefTypeAlign(Type *Ty) const;

/// Returns an integer type with size at least as big as that of a
/// pointer in the given address space.
IntegerType *getIntPtrType(LLVMContext &C, unsigned AddressSpace = 0) const;

/// Returns an integer (vector of integer) type with size at least as
/// big as that of a pointer of the given pointer (vector of pointer) type.
Type *getIntPtrType(Type *) const;

/// Returns the smallest integer type with size at least as big as
/// Width bits.
Type *getSmallestLegalIntType(LLVMContext &C, unsigned Width = 0) const;

/// Returns the largest legal integer type, or null if none are set.
Type *getLargestLegalIntType(LLVMContext &C) const {
  unsigned LargestSize = getLargestLegalIntTypeSizeInBits();
  return (LargestSize == 0) ? nullptr : Type::getIntNTy(C, LargestSize);
}

/// Returns the size of largest legal integer type size, or 0 if none
/// are set.
unsigned getLargestLegalIntTypeSizeInBits() const;

/// Returns the type of a GEP index.
/// If it was not specified explicitly, it will be the integer type of the
/// pointer width - IntPtrType.
Type *getIndexType(Type *PtrTy) const;

/// Returns the offset from the beginning of the type for the specified
/// indices.
///
/// Note that this takes the element type, not the pointer type.
/// This is used to implement getelementptr.
int64_t getIndexedOffsetInType(Type *ElemTy, ArrayRef<Value *> Indices) const;

/// Get GEP indices to access Offset inside ElemTy. ElemTy is updated to be
/// the result element type and Offset to be the residual offset.
SmallVector<APInt> getGEPIndicesForOffset(Type *&ElemTy, APInt &Offset) const;

/// Returns a StructLayout object, indicating the alignment of the
/// struct, its size, and the offsets of its fields.
///
/// Note that this information is lazily cached.
const StructLayout *getStructLayout(StructType *Ty) const;

/// Returns the preferred alignment of the specified global.
///
/// This includes an explicitly requested alignment (if the global has one).
Align getPreferredAlign(const GlobalVariable *GV) const;
601};

603inline DataLayout *unwrap(LLVMTargetDataRef P) {
return reinterpret_cast<DataLayout *>(P);
605}

607inline LLVMTargetDataRef wrap(const DataLayout *P) {
return reinterpret_cast<LLVMTargetDataRef>(const_cast<DataLayout *>(P));
609}

611/// Used to lazily calculate structure layout information for a target machine,
612/// based on the DataLayout structure.
613class StructLayout final : public TrailingObjects<StructLayout, uint64_t> {
uint64_t StructSize;
Align StructAlignment;
unsigned IsPadded : 1;
unsigned NumElements : 31;

619public:
uint64_t getSizeInBytes() const { return StructSize; }

uint64_t getSizeInBits() const { return 8 * StructSize; }

Align getAlignment() const { return StructAlignment; }

/// Returns whether the struct has padding or not between its fields.
/// NB: Padding in nested element is not taken into account.
bool hasPadding() const { return IsPadded; }

/// Given a valid byte offset into the structure, returns the structure
/// index that contains it.
unsigned getElementContainingOffset(uint64_t Offset) const;

MutableArrayRef<uint64_t> getMemberOffsets() {
  return llvm::makeMutableArrayRef(getTrailingObjects<uint64_t>(),
                                   NumElements);
}

ArrayRef<uint64_t> getMemberOffsets() const {
  return llvm::makeArrayRef(getTrailingObjects<uint64_t>(), NumElements);
}

uint64_t getElementOffset(unsigned Idx) const {
  assert(Idx < NumElements && "Invalid element idx!")(static_cast <bool> (Idx < NumElements && "Invalid element idx!"
) ? void (0) : __assert_fail ("Idx < NumElements && \"Invalid element idx!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h"
, 644, __extension__ __PRETTY_FUNCTION__));
  return getMemberOffsets()[Idx];
}

uint64_t getElementOffsetInBits(unsigned Idx) const {
  return getElementOffset(Idx) * 8;
}

652private:
friend class DataLayout; // Only DataLayout can create this class

StructLayout(StructType *ST, const DataLayout &DL);

size_t numTrailingObjects(OverloadToken<uint64_t>) const {
  return NumElements;
}
660};

662// The implementation of this method is provided inline as it is particularly
663// well suited to constant folding when called on a specific Type subclass.
664inline TypeSize DataLayout::getTypeSizeInBits(Type *Ty) const {
assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!")(static_cast <bool> (Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!"
) ? void (0) : __assert_fail ("Ty->isSized() && \"Cannot getTypeInfo() on a type that is unsized!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h"
, 665, __extension__ __PRETTY_FUNCTION__));
switch (Ty->getTypeID()) {
case Type::LabelTyID:
  return TypeSize::Fixed(getPointerSizeInBits(0));
case Type::PointerTyID:
  return TypeSize::Fixed(getPointerSizeInBits(Ty->getPointerAddressSpace()));
case Type::ArrayTyID: {
  ArrayType *ATy = cast<ArrayType>(Ty);
  return ATy->getNumElements() *
         getTypeAllocSizeInBits(ATy->getElementType());
}
case Type::StructTyID:
  // Get the layout annotation... which is lazily created on demand.
  return TypeSize::Fixed(
                      getStructLayout(cast<StructType>(Ty))->getSizeInBits());
case Type::IntegerTyID:
  return TypeSize::Fixed(Ty->getIntegerBitWidth());
case Type::HalfTyID:
case Type::BFloatTyID:
  return TypeSize::Fixed(16);
case Type::FloatTyID:
  return TypeSize::Fixed(32);
case Type::DoubleTyID:
case Type::X86_MMXTyID:
  return TypeSize::Fixed(64);
case Type::PPC_FP128TyID:
case Type::FP128TyID:
  return TypeSize::Fixed(128);
case Type::X86_AMXTyID:
  return TypeSize::Fixed(8192);
// In memory objects this is always aligned to a higher boundary, but
// only 80 bits contain information.
case Type::X86_FP80TyID:
  return TypeSize::Fixed(80);
case Type::FixedVectorTyID:
case Type::ScalableVectorTyID: {
  VectorType *VTy = cast<VectorType>(Ty);
  auto EltCnt = VTy->getElementCount();
  uint64_t MinBits = EltCnt.getKnownMinValue() *
                     getTypeSizeInBits(VTy->getElementType()).getFixedSize();
  return TypeSize(MinBits, EltCnt.isScalable());
}
default:
  llvm_unreachable("DataLayout::getTypeSizeInBits(): Unsupported type")::llvm::llvm_unreachable_internal("DataLayout::getTypeSizeInBits(): Unsupported type"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/IR/DataLayout.h"
, 708);
}
710}

712} // end namespace llvm

714#endif // LLVM_IR_DATALAYOUT_H