/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/APInt.h

Bug Summary

File:	llvm/include/llvm/ADT/APInt.h
Warning:	line 444, column 36 The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'llvm::APInt::WordType'
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name TargetLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-09-002921-48462-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

→
1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//

13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/CodeGen/CallingConvLower.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineFunction.h"
18#include "llvm/CodeGen/MachineJumpTableInfo.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/CodeGen/SelectionDAG.h"
21#include "llvm/CodeGen/TargetRegisterInfo.h"
22#include "llvm/CodeGen/TargetSubtargetInfo.h"
23#include "llvm/IR/DataLayout.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/IR/LLVMContext.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/Support/ErrorHandling.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/Target/TargetLoweringObjectFile.h"
33#include "llvm/Target/TargetMachine.h"
34#include <cctype>
35using namespace llvm;

37/// NOTE: The TargetMachine owns TLOF.
38TargetLowering::TargetLowering(const TargetMachine &tm)
  : TargetLoweringBase(tm) {}

41const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
return nullptr;
43}

45bool TargetLowering::isPositionIndependent() const {
return getTargetMachine().isPositionIndependent();
47}

49/// Check whether a given call node is in tail position within its function. If
50/// so, it sets Chain to the input chain of the tail call.
51bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
                                        SDValue &Chain) const {
const Function &F = DAG.getMachineFunction().getFunction();

// Conservatively require the attributes of the call to match those of
// the return. Ignore NoAlias and NonNull because they don't affect the
// call sequence.
AttributeList CallerAttrs = F.getAttributes();
if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
        .removeAttribute(Attribute::NoAlias)
        .removeAttribute(Attribute::NonNull)
        .hasAttributes())
  return false;

// It's not safe to eliminate the sign / zero extension of the return value.
if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
    CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
  return false;

// Check if the only use is a function return node.
return isUsedByReturnOnly(Node, Chain);
72}

74bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
  const uint32_t *CallerPreservedMask,
  const SmallVectorImpl<CCValAssign> &ArgLocs,
  const SmallVectorImpl<SDValue> &OutVals) const {
for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
  const CCValAssign &ArgLoc = ArgLocs[I];
  if (!ArgLoc.isRegLoc())
    continue;
  Register Reg = ArgLoc.getLocReg();
  // Only look at callee saved registers.
  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
    continue;
  // Check that we pass the value used for the caller.
  // (We look for a CopyFromReg reading a virtual register that is used
  //  for the function live-in value of register Reg)
  SDValue Value = OutVals[I];
  if (Value->getOpcode() != ISD::CopyFromReg)
    return false;
  unsigned ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
    return false;
}
return true;
97}

99/// Set CallLoweringInfo attribute flags based on a call instruction
100/// and called function attributes.
101void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
                                                   unsigned ArgIdx) {
IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
Alignment = Call->getParamAlignment(ArgIdx);
ByValType = nullptr;
if (Call->paramHasAttr(ArgIdx, Attribute::ByVal))
  ByValType = Call->getParamByValType(ArgIdx);
117}

119/// Generate a libcall taking the given operands as arguments and returning a
120/// result of type RetVT.
121std::pair<SDValue, SDValue>
122TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
                          ArrayRef<SDValue> Ops,
                          MakeLibCallOptions CallOptions,
                          const SDLoc &dl) const {
TargetLowering::ArgListTy Args;
Args.reserve(Ops.size());

TargetLowering::ArgListEntry Entry;
for (unsigned i = 0; i < Ops.size(); ++i) {
  SDValue NewOp = Ops[i];
  Entry.Node = NewOp;
  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
  Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
                                               CallOptions.IsSExt);
  Entry.IsZExt = !Entry.IsSExt;

  if (CallOptions.IsSoften &&
      !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
    Entry.IsSExt = Entry.IsZExt = false;
  }
  Args.push_back(Entry);
}

if (LC == RTLIB::UNKNOWN_LIBCALL)
  report_fatal_error("Unsupported library call operation!");
SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
                                       getPointerTy(DAG.getDataLayout()));

Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
TargetLowering::CallLoweringInfo CLI(DAG);
bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
bool zeroExtend = !signExtend;

if (CallOptions.IsSoften &&
    !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
  signExtend = zeroExtend = false;
}

CLI.setDebugLoc(dl)
    .setChain(DAG.getEntryNode())
    .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
    .setNoReturn(CallOptions.DoesNotReturn)
    .setDiscardResult(!CallOptions.IsReturnValueUsed)
    .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
    .setSExtResult(signExtend)
    .setZExtResult(zeroExtend);
return LowerCallTo(CLI);
169}

171bool
172TargetLowering::findOptimalMemOpLowering(std::vector<EVT> &MemOps,
                                       unsigned Limit, uint64_t Size,
                                       unsigned DstAlign, unsigned SrcAlign,
                                       bool IsMemset,
                                       bool ZeroMemset,
                                       bool MemcpyStrSrc,
                                       bool AllowOverlap,
                                       unsigned DstAS, unsigned SrcAS,
                                       const AttributeList &FuncAttributes) const {
// If 'SrcAlign' is zero, that means the memory operation does not need to
// load the value, i.e. memset or memcpy from constant string. Otherwise,
// it's the inferred alignment of the source. 'DstAlign', on the other hand,
// is the specified alignment of the memory operation. If it is zero, that
// means it's possible to change the alignment of the destination.
// 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
// not need to be loaded.
if (!(SrcAlign == 0 || SrcAlign >= DstAlign))
  return false;

EVT VT = getOptimalMemOpType(Size, DstAlign, SrcAlign,
                             IsMemset, ZeroMemset, MemcpyStrSrc,
                             FuncAttributes);

if (VT == MVT::Other) {
  // Use the largest integer type whose alignment constraints are satisfied.
  // We only need to check DstAlign here as SrcAlign is always greater or
  // equal to DstAlign (or zero).
  VT = MVT::i64;
  while (DstAlign && DstAlign < VT.getSizeInBits() / 8 &&
         !allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
    VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
  assert(VT.isInteger())((VT.isInteger()) ? static_cast<void> (0) : __assert_fail
 ("VT.isInteger()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 203, __PRETTY_FUNCTION__));

  // Find the largest legal integer type.
  MVT LVT = MVT::i64;
  while (!isTypeLegal(LVT))
    LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
  assert(LVT.isInteger())((LVT.isInteger()) ? static_cast<void> (0) : __assert_fail
 ("LVT.isInteger()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 209, __PRETTY_FUNCTION__));

  // If the type we've chosen is larger than the largest legal integer type
  // then use that instead.
  if (VT.bitsGT(LVT))
    VT = LVT;
}

unsigned NumMemOps = 0;
while (Size != 0) {
  unsigned VTSize = VT.getSizeInBits() / 8;
  while (VTSize > Size) {
    // For now, only use non-vector load / store's for the left-over pieces.
    EVT NewVT = VT;
    unsigned NewVTSize;

    bool Found = false;
    if (VT.isVector() || VT.isFloatingPoint()) {
      NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
      if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
          isSafeMemOpType(NewVT.getSimpleVT()))
        Found = true;
      else if (NewVT == MVT::i64 &&
               isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
               isSafeMemOpType(MVT::f64)) {
        // i64 is usually not legal on 32-bit targets, but f64 may be.
        NewVT = MVT::f64;
        Found = true;
      }
    }

    if (!Found) {
      do {
        NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
        if (NewVT == MVT::i8)
          break;
      } while (!isSafeMemOpType(NewVT.getSimpleVT()));
    }
    NewVTSize = NewVT.getSizeInBits() / 8;

    // If the new VT cannot cover all of the remaining bits, then consider
    // issuing a (or a pair of) unaligned and overlapping load / store.
    bool Fast;
    if (NumMemOps && AllowOverlap && NewVTSize < Size &&
        allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign,
                                       MachineMemOperand::MONone, &Fast) &&
        Fast)
      VTSize = Size;
    else {
      VT = NewVT;
      VTSize = NewVTSize;
    }
  }

  if (++NumMemOps > Limit)
    return false;

  MemOps.push_back(VT);
  Size -= VTSize;
}

return true;
271}

273/// Soften the operands of a comparison. This code is shared among BR_CC,
274/// SELECT_CC, and SETCC handlers.
275void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
                                       SDValue &NewLHS, SDValue &NewRHS,
                                       ISD::CondCode &CCCode,
                                       const SDLoc &dl, const SDValue OldLHS,
                                       const SDValue OldRHS) const {
assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
 MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 281, __PRETTY_FUNCTION__))
       && "Unsupported setcc type!")(((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT ==
 MVT::ppcf128) && "Unsupported setcc type!") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128) && \"Unsupported setcc type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 281, __PRETTY_FUNCTION__));

// Expand into one or more soft-fp libcall(s).
RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
bool ShouldInvertCC = false;
switch (CCCode) {
case ISD::SETEQ:
case ISD::SETOEQ:
  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
        (VT == MVT::f64) ? RTLIB::OEQ_F64 :
        (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
  break;
case ISD::SETNE:
case ISD::SETUNE:
  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
        (VT == MVT::f64) ? RTLIB::UNE_F64 :
        (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
  break;
case ISD::SETGE:
case ISD::SETOGE:
  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
        (VT == MVT::f64) ? RTLIB::OGE_F64 :
        (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
  break;
case ISD::SETLT:
case ISD::SETOLT:
  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
        (VT == MVT::f64) ? RTLIB::OLT_F64 :
        (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
  break;
case ISD::SETLE:
case ISD::SETOLE:
  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
        (VT == MVT::f64) ? RTLIB::OLE_F64 :
        (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
  break;
case ISD::SETGT:
case ISD::SETOGT:
  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
        (VT == MVT::f64) ? RTLIB::OGT_F64 :
        (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
  break;
case ISD::SETUO:
  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
        (VT == MVT::f64) ? RTLIB::UO_F64 :
        (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
  break;
case ISD::SETO:
  LC1 = (VT == MVT::f32) ? RTLIB::O_F32 :
        (VT == MVT::f64) ? RTLIB::O_F64 :
        (VT == MVT::f128) ? RTLIB::O_F128 : RTLIB::O_PPCF128;
  break;
case ISD::SETONE:
  // SETONE = SETOLT | SETOGT
  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
        (VT == MVT::f64) ? RTLIB::OLT_F64 :
        (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
  LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
        (VT == MVT::f64) ? RTLIB::OGT_F64 :
        (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
  break;
case ISD::SETUEQ:
  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
        (VT == MVT::f64) ? RTLIB::UO_F64 :
        (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
        (VT == MVT::f64) ? RTLIB::OEQ_F64 :
        (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
  break;
default:
  // Invert CC for unordered comparisons
  ShouldInvertCC = true;
  switch (CCCode) {
  case ISD::SETULT:
    LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
          (VT == MVT::f64) ? RTLIB::OGE_F64 :
          (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
    break;
  case ISD::SETULE:
    LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
          (VT == MVT::f64) ? RTLIB::OGT_F64 :
          (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
    break;
  case ISD::SETUGT:
    LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
          (VT == MVT::f64) ? RTLIB::OLE_F64 :
          (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
    break;
  case ISD::SETUGE:
    LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
          (VT == MVT::f64) ? RTLIB::OLT_F64 :
          (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
    break;
  default: llvm_unreachable("Do not know how to soften this setcc!")::llvm::llvm_unreachable_internal("Do not know how to soften this setcc!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 374);
  }
}

// Use the target specific return value for comparions lib calls.
EVT RetVT = getCmpLibcallReturnType();
SDValue Ops[2] = {NewLHS, NewRHS};
TargetLowering::MakeLibCallOptions CallOptions;
EVT OpsVT[2] = { OldLHS.getValueType(),
                 OldRHS.getValueType() };
CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
NewLHS = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl).first;
NewRHS = DAG.getConstant(0, dl, RetVT);

CCCode = getCmpLibcallCC(LC1);
if (ShouldInvertCC)
  CCCode = getSetCCInverse(CCCode, /*isInteger=*/true);

if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
  SDValue Tmp = DAG.getNode(
      ISD::SETCC, dl,
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
      NewLHS, NewRHS, DAG.getCondCode(CCCode));
  NewLHS = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl).first;
  NewLHS = DAG.getNode(
      ISD::SETCC, dl,
      getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT),
      NewLHS, NewRHS, DAG.getCondCode(getCmpLibcallCC(LC2)));
  NewLHS = DAG.getNode(ISD::OR, dl, Tmp.getValueType(), Tmp, NewLHS);
  NewRHS = SDValue();
}
405}

407/// Return the entry encoding for a jump table in the current function. The
408/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
409unsigned TargetLowering::getJumpTableEncoding() const {
// In non-pic modes, just use the address of a block.
if (!isPositionIndependent())
  return MachineJumpTableInfo::EK_BlockAddress;

// In PIC mode, if the target supports a GPRel32 directive, use it.
if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
  return MachineJumpTableInfo::EK_GPRel32BlockAddress;

// Otherwise, use a label difference.
return MachineJumpTableInfo::EK_LabelDifference32;
420}

422SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
                                               SelectionDAG &DAG) const {
// If our PIC model is GP relative, use the global offset table as the base.
unsigned JTEncoding = getJumpTableEncoding();

if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
    (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
  return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));

return Table;
432}

434/// This returns the relocation base for the given PIC jumptable, the same as
435/// getPICJumpTableRelocBase, but as an MCExpr.
436const MCExpr *
437TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
                                           unsigned JTI,MCContext &Ctx) const{
// The normal PIC reloc base is the label at the start of the jump table.
return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
441}

443bool
444TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
const TargetMachine &TM = getTargetMachine();
const GlobalValue *GV = GA->getGlobal();

// If the address is not even local to this DSO we will have to load it from
// a got and then add the offset.
if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
  return false;

// If the code is position independent we will have to add a base register.
if (isPositionIndependent())
  return false;

// Otherwise we can do it.
return true;
459}

461//===----------------------------------------------------------------------===//
462//  Optimization Methods
463//===----------------------------------------------------------------------===//

465/// If the specified instruction has a constant integer operand and there are
466/// bits set in that constant that are not demanded, then clear those bits and
467/// return true.
468bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
                                          TargetLoweringOpt &TLO) const {
SDLoc DL(Op);
unsigned Opcode = Op.getOpcode();

// Do target-specific constant optimization.
if (targetShrinkDemandedConstant(Op, Demanded, TLO))
  return TLO.New.getNode();

// FIXME: ISD::SELECT, ISD::SELECT_CC
switch (Opcode) {
default:
  break;
case ISD::XOR:
case ISD::AND:
case ISD::OR: {
  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  if (!Op1C)
    return false;

  // If this is a 'not' op, don't touch it because that's a canonical form.
  const APInt &C = Op1C->getAPIntValue();
  if (Opcode == ISD::XOR && Demanded.isSubsetOf(C))
    return false;

  if (!C.isSubsetOf(Demanded)) {
    EVT VT = Op.getValueType();
    SDValue NewC = TLO.DAG.getConstant(Demanded & C, DL, VT);
    SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
    return TLO.CombineTo(Op, NewOp);
  }

  break;
}
}

return false;
505}

507/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
508/// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
509/// generalized for targets with other types of implicit widening casts.
510bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
                                    const APInt &Demanded,
                                    TargetLoweringOpt &TLO) const {
assert(Op.getNumOperands() == 2 &&((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 514, __PRETTY_FUNCTION__))
       "ShrinkDemandedOp only supports binary operators!")((Op.getNumOperands() == 2 && "ShrinkDemandedOp only supports binary operators!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 2 && \"ShrinkDemandedOp only supports binary operators!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 514, __PRETTY_FUNCTION__));
assert(Op.getNode()->getNumValues() == 1 &&((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 516, __PRETTY_FUNCTION__))
       "ShrinkDemandedOp only supports nodes with one result!")((Op.getNode()->getNumValues() == 1 && "ShrinkDemandedOp only supports nodes with one result!"
) ? static_cast<void> (0) : __assert_fail ("Op.getNode()->getNumValues() == 1 && \"ShrinkDemandedOp only supports nodes with one result!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 516, __PRETTY_FUNCTION__));

SelectionDAG &DAG = TLO.DAG;
SDLoc dl(Op);

// Early return, as this function cannot handle vector types.
if (Op.getValueType().isVector())
  return false;

// Don't do this if the node has another user, which may require the
// full value.
if (!Op.getNode()->hasOneUse())
  return false;

// Search for the smallest integer type with free casts to and from
// Op's type. For expedience, just check power-of-2 integer types.
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
unsigned DemandedSize = Demanded.getActiveBits();
unsigned SmallVTBits = DemandedSize;
if (!isPowerOf2_32(SmallVTBits))
  SmallVTBits = NextPowerOf2(SmallVTBits);
for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
      TLI.isZExtFree(SmallVT, Op.getValueType())) {
    // We found a type with free casts.
    SDValue X = DAG.getNode(
        Op.getOpcode(), dl, SmallVT,
        DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
        DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
    assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?")((DemandedSize <= SmallVTBits && "Narrowed below demanded bits?"
) ? static_cast<void> (0) : __assert_fail ("DemandedSize <= SmallVTBits && \"Narrowed below demanded bits?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 546, __PRETTY_FUNCTION__));
    SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
    return TLO.CombineTo(Op, Z);
  }
}
return false;
552}

554bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                        DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                      !DCI.isBeforeLegalizeOps());
KnownBits Known;

bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
if (Simplified) {
  DCI.AddToWorklist(Op.getNode());
  DCI.CommitTargetLoweringOpt(TLO);
}
return Simplified;
567}

569bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                        KnownBits &Known,
                                        TargetLoweringOpt &TLO,
                                        unsigned Depth,
                                        bool AssumeSingleUse) const {
EVT VT = Op.getValueType();
APInt DemandedElts = VT.isVector()
                         ? APInt::getAllOnesValue(VT.getVectorNumElements())
                         : APInt(1, 1);
return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
                            AssumeSingleUse);
580}

582// TODO: Can we merge SelectionDAG::GetDemandedBits into this?
583// TODO: Under what circumstances can we create nodes? Constant folding?
584SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  SelectionDAG &DAG, unsigned Depth) const {
// Limit search depth.
if (Depth >= SelectionDAG::MaxRecursionDepth)
  return SDValue();

// Ignore UNDEFs.
if (Op.isUndef())
  return SDValue();

// Not demanding any bits/elts from Op.
if (DemandedBits == 0 || DemandedElts == 0)
  return DAG.getUNDEF(Op.getValueType());

unsigned NumElts = DemandedElts.getBitWidth();
KnownBits LHSKnown, RHSKnown;
switch (Op.getOpcode()) {
case ISD::BITCAST: {
  SDValue Src = peekThroughBitcasts(Op.getOperand(0));
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Op.getValueType();
  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
  unsigned NumDstEltBits = DstVT.getScalarSizeInBits();

  if (NumSrcEltBits == NumDstEltBits)
    if (SDValue V = SimplifyMultipleUseDemandedBits(
            Src, DemandedBits, DemandedElts, DAG, Depth + 1))
      return DAG.getBitcast(DstVT, V);

  // TODO - bigendian once we have test coverage.
  if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
      DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = NumDstEltBits / NumSrcEltBits;
    unsigned NumSrcElts = SrcVT.getVectorNumElements();
    APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
    for (unsigned i = 0; i != Scale; ++i) {
      unsigned Offset = i * NumSrcEltBits;
      APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
      if (!Sub.isNullValue()) {
        DemandedSrcBits |= Sub;
        for (unsigned j = 0; j != NumElts; ++j)
          if (DemandedElts[j])
            DemandedSrcElts.setBit((j * Scale) + i);
      }
    }

    if (SDValue V = SimplifyMultipleUseDemandedBits(
            Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
      return DAG.getBitcast(DstVT, V);
  }

  // TODO - bigendian once we have test coverage.
  if ((NumSrcEltBits % NumDstEltBits) == 0 &&
      DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = NumSrcEltBits / NumDstEltBits;
    unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
    APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
    for (unsigned i = 0; i != NumElts; ++i)
      if (DemandedElts[i]) {
        unsigned Offset = (i % Scale) * NumDstEltBits;
        DemandedSrcBits.insertBits(DemandedBits, Offset);
        DemandedSrcElts.setBit(i / Scale);
      }

    if (SDValue V = SimplifyMultipleUseDemandedBits(
            Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
      return DAG.getBitcast(DstVT, V);
  }

  break;
}
case ISD::AND: {
  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

  // If all of the demanded bits are known 1 on one side, return the other.
  // These bits cannot contribute to the result of the 'and' in this
  // context.
  if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
    return Op.getOperand(0);
  if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
    return Op.getOperand(1);
  break;
}
case ISD::OR: {
  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

  // If all of the demanded bits are known zero on one side, return the
  // other.  These bits cannot contribute to the result of the 'or' in this
  // context.
  if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
    return Op.getOperand(0);
  if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
    return Op.getOperand(1);
  break;
}
case ISD::XOR: {
  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);

  // If all of the demanded bits are known zero on one side, return the
  // other.
  if (DemandedBits.isSubsetOf(RHSKnown.Zero))
    return Op.getOperand(0);
  if (DemandedBits.isSubsetOf(LHSKnown.Zero))
    return Op.getOperand(1);
  break;
}
case ISD::SIGN_EXTEND_INREG: {
  // If none of the extended bits are demanded, eliminate the sextinreg.
  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  if (DemandedBits.getActiveBits() <= ExVT.getScalarSizeInBits())
    return Op.getOperand(0);
  break;
}
case ISD::INSERT_VECTOR_ELT: {
  // If we don't demand the inserted element, return the base vector.
  SDValue Vec = Op.getOperand(0);
  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  EVT VecVT = Vec.getValueType();
  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
      !DemandedElts[CIdx->getZExtValue()])
    return Vec;
  break;
}
case ISD::VECTOR_SHUFFLE: {
  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

  // If all the demanded elts are from one operand and are inline,
  // then we can use the operand directly.
  bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = ShuffleMask[i];
    if (M < 0 || !DemandedElts[i])
      continue;
    AllUndef = false;
    IdentityLHS &= (M == (int)i);
    IdentityRHS &= ((M - NumElts) == i);
  }

  if (AllUndef)
    return DAG.getUNDEF(Op.getValueType());
  if (IdentityLHS)
    return Op.getOperand(0);
  if (IdentityRHS)
    return Op.getOperand(1);
  break;
}
default:
  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
    if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
            Op, DemandedBits, DemandedElts, DAG, Depth))
      return V;
  break;
}
return SDValue();
744}

746/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
747/// result of Op are ever used downstream. If we can use this information to
748/// simplify Op, create a new simplified DAG node and return true, returning the
749/// original and new nodes in Old and New. Otherwise, analyze the expression and
750/// return a mask of Known bits for the expression (used to simplify the
751/// caller).  The Known bits may only be accurate for those bits in the
752/// OriginalDemandedBits and OriginalDemandedElts.
753bool TargetLowering::SimplifyDemandedBits(
  SDValue Op, const APInt &OriginalDemandedBits,
  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
  unsigned Depth, bool AssumeSingleUse) const {
unsigned BitWidth = OriginalDemandedBits.getBitWidth();
assert(Op.getScalarValueSizeInBits() == BitWidth &&((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 759, __PRETTY_FUNCTION__))
       "Mask size mismatches value type size!")((Op.getScalarValueSizeInBits() == BitWidth && "Mask size mismatches value type size!"
) ? static_cast<void> (0) : __assert_fail ("Op.getScalarValueSizeInBits() == BitWidth && \"Mask size mismatches value type size!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 759, __PRETTY_FUNCTION__));

unsigned NumElts = OriginalDemandedElts.getBitWidth();
assert((!Op.getValueType().isVector() ||(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 764, __PRETTY_FUNCTION__))
        NumElts == Op.getValueType().getVectorNumElements()) &&(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 764, __PRETTY_FUNCTION__))
       "Unexpected vector size")(((!Op.getValueType().isVector() || NumElts == Op.getValueType
().getVectorNumElements()) && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("(!Op.getValueType().isVector() || NumElts == Op.getValueType().getVectorNumElements()) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 764, __PRETTY_FUNCTION__));

APInt DemandedBits = OriginalDemandedBits;
APInt DemandedElts = OriginalDemandedElts;
SDLoc dl(Op);
auto &DL = TLO.DAG.getDataLayout();

// Don't know anything.
Known = KnownBits(BitWidth);

// Undef operand.
if (Op.isUndef())
  return false;

if (Op.getOpcode() == ISD::Constant) {
  // We know all of the bits for a constant!
  Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
  Known.Zero = ~Known.One;
  return false;
}

// Other users may use these bits.
EVT VT = Op.getValueType();
if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
  if (Depth != 0) {
    // If not at the root, Just compute the Known bits to
    // simplify things downstream.
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false;
  }
  // If this is the root being simplified, allow it to have multiple uses,
  // just set the DemandedBits/Elts to all bits.
  DemandedBits = APInt::getAllOnesValue(BitWidth);
  DemandedElts = APInt::getAllOnesValue(NumElts);
} else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
  // Not demanding any bits/elts from Op.
  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
} else if (Depth >= SelectionDAG::MaxRecursionDepth) {
  // Limit search depth.
  return false;
}

KnownBits Known2, KnownOut;
switch (Op.getOpcode()) {
case ISD::TargetConstant:
  llvm_unreachable("Can't simplify this node")::llvm::llvm_unreachable_internal("Can't simplify this node",
 "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 809);
case ISD::SCALAR_TO_VECTOR: {
  if (!DemandedElts[0])
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

  KnownBits SrcKnown;
  SDValue Src = Op.getOperand(0);
  unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
  APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
    return true;
  Known = SrcKnown.zextOrTrunc(BitWidth, false);
  break;
}
case ISD::BUILD_VECTOR:
  // Collect the known bits that are shared by every demanded element.
  // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
  return false; // Don't fall through, will infinitely loop.
case ISD::LOAD: {
  LoadSDNode *LD = cast<LoadSDNode>(Op);
  if (getTargetConstantFromLoad(LD)) {
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false; // Don't fall through, will infinitely loop.
  }
  break;
}
case ISD::INSERT_VECTOR_ELT: {
  SDValue Vec = Op.getOperand(0);
  SDValue Scl = Op.getOperand(1);
  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
  EVT VecVT = Vec.getValueType();

  // If index isn't constant, assume we need all vector elements AND the
  // inserted element.
  APInt DemandedVecElts(DemandedElts);
  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
    unsigned Idx = CIdx->getZExtValue();
    DemandedVecElts.clearBit(Idx);

    // Inserted element is not required.
    if (!DemandedElts[Idx])
      return TLO.CombineTo(Op, Vec);
  }

  KnownBits KnownScl;
  unsigned NumSclBits = Scl.getScalarValueSizeInBits();
  APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
  if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
    return true;

  Known = KnownScl.zextOrTrunc(BitWidth, false);

  KnownBits KnownVec;
  if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
                           Depth + 1))
    return true;

  if (!!DemandedVecElts) {
    Known.One &= KnownVec.One;
    Known.Zero &= KnownVec.Zero;
  }

  return false;
}
case ISD::INSERT_SUBVECTOR: {
  SDValue Base = Op.getOperand(0);
  SDValue Sub = Op.getOperand(1);
  EVT SubVT = Sub.getValueType();
  unsigned NumSubElts = SubVT.getVectorNumElements();

  // If index isn't constant, assume we need the original demanded base
  // elements and ALL the inserted subvector elements.
  APInt BaseElts = DemandedElts;
  APInt SubElts = APInt::getAllOnesValue(NumSubElts);
  if (isa<ConstantSDNode>(Op.getOperand(2))) {
    const APInt &Idx = Op.getConstantOperandAPInt(2);
    if (Idx.ule(NumElts - NumSubElts)) {
      unsigned SubIdx = Idx.getZExtValue();
      SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
      BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);
    }
  }

  KnownBits KnownSub, KnownBase;
  if (SimplifyDemandedBits(Sub, DemandedBits, SubElts, KnownSub, TLO,
                           Depth + 1))
    return true;
  if (SimplifyDemandedBits(Base, DemandedBits, BaseElts, KnownBase, TLO,
                           Depth + 1))
    return true;

  Known.Zero.setAllBits();
  Known.One.setAllBits();
  if (!!SubElts) {
      Known.One &= KnownSub.One;
      Known.Zero &= KnownSub.Zero;
  }
  if (!!BaseElts) {
      Known.One &= KnownBase.One;
      Known.Zero &= KnownBase.Zero;
  }
  break;
}
case ISD::EXTRACT_SUBVECTOR: {
  // If index isn't constant, assume we need all the source vector elements.
  SDValue Src = Op.getOperand(0);
  ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  APInt SrcElts = APInt::getAllOnesValue(NumSrcElts);
  if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
    // Offset the demanded elts by the subvector index.
    uint64_t Idx = SubIdx->getZExtValue();
    SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
  }
  if (SimplifyDemandedBits(Src, DemandedBits, SrcElts, Known, TLO, Depth + 1))
    return true;
  break;
}
case ISD::CONCAT_VECTORS: {
  Known.Zero.setAllBits();
  Known.One.setAllBits();
  EVT SubVT = Op.getOperand(0).getValueType();
  unsigned NumSubVecs = Op.getNumOperands();
  unsigned NumSubElts = SubVT.getVectorNumElements();
  for (unsigned i = 0; i != NumSubVecs; ++i) {
    APInt DemandedSubElts =
        DemandedElts.extractBits(NumSubElts, i * NumSubElts);
    if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
                             Known2, TLO, Depth + 1))
      return true;
    // Known bits are shared by every demanded subvector element.
    if (!!DemandedSubElts) {
      Known.One &= Known2.One;
      Known.Zero &= Known2.Zero;
    }
  }
  break;
}
case ISD::VECTOR_SHUFFLE: {
  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

  // Collect demanded elements from shuffle operands..
  APInt DemandedLHS(NumElts, 0);
  APInt DemandedRHS(NumElts, 0);
  for (unsigned i = 0; i != NumElts; ++i) {
    if (!DemandedElts[i])
      continue;
    int M = ShuffleMask[i];
    if (M < 0) {
      // For UNDEF elements, we don't know anything about the common state of
      // the shuffle result.
      DemandedLHS.clearAllBits();
      DemandedRHS.clearAllBits();
      break;
    }
    assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 965, __PRETTY_FUNCTION__));
    if (M < (int)NumElts)
      DemandedLHS.setBit(M);
    else
      DemandedRHS.setBit(M - NumElts);
  }

  if (!!DemandedLHS || !!DemandedRHS) {
    SDValue Op0 = Op.getOperand(0);
    SDValue Op1 = Op.getOperand(1);

    Known.Zero.setAllBits();
    Known.One.setAllBits();
    if (!!DemandedLHS) {
      if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
                               Depth + 1))
        return true;
      Known.One &= Known2.One;
      Known.Zero &= Known2.Zero;
    }
    if (!!DemandedRHS) {
      if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
                               Depth + 1))
        return true;
      Known.One &= Known2.One;
      Known.Zero &= Known2.Zero;
    }

    // Attempt to avoid multi-use ops if we don't need anything from them.
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
      return TLO.CombineTo(Op, NewOp);
    }
  }
  break;
}
case ISD::AND: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  // If the RHS is a constant, check to see if the LHS would be zero without
  // using the bits from the RHS.  Below, we use knowledge about the RHS to
  // simplify the LHS, here we're using information from the LHS to simplify
  // the RHS.
  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
    // Do not increment Depth here; that can cause an infinite loop.
    KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
    // If the LHS already has zeros where RHSC does, this 'and' is dead.
    if ((LHSKnown.Zero & DemandedBits) ==
        (~RHSC->getAPIntValue() & DemandedBits))
      return TLO.CombineTo(Op, Op0);

    // If any of the set bits in the RHS are known zero on the LHS, shrink
    // the constant.
    if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits, TLO))
      return true;

    // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
    // constant, but if this 'and' is only clearing bits that were just set by
    // the xor, then this 'and' can be eliminated by shrinking the mask of
    // the xor. For example, for a 32-bit X:
    // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
    if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
        LHSKnown.One == ~RHSC->getAPIntValue()) {
      SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
      return TLO.CombineTo(Op, Xor);
    }
  }

  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1043, __PRETTY_FUNCTION__));
  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
                           Known2, TLO, Depth + 1))
    return true;
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1047, __PRETTY_FUNCTION__));

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If all of the demanded bits are known one on one side, return the other.
  // These bits cannot contribute to the result of the 'and'.
  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
    return TLO.CombineTo(Op, Op0);
  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
    return TLO.CombineTo(Op, Op1);
  // If all of the demanded bits in the inputs are known zeros, return zero.
  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
    return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
  // If the RHS is a constant, see if we can simplify it.
  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, TLO))
    return true;
  // If the operation can be done in a smaller type, do so.
  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
    return true;

  // Output known-1 bits are only known if set in both the LHS & RHS.
  Known.One &= Known2.One;
  // Output known-0 are known to be clear if zero in either the LHS | RHS.
  Known.Zero |= Known2.Zero;
  break;
}
case ISD::OR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1092, __PRETTY_FUNCTION__));
  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
                           Known2, TLO, Depth + 1))
    return true;
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1096, __PRETTY_FUNCTION__));

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If all of the demanded bits are known zero on one side, return the other.
  // These bits cannot contribute to the result of the 'or'.
  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
    return TLO.CombineTo(Op, Op0);
  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
    return TLO.CombineTo(Op, Op1);
  // If the RHS is a constant, see if we can simplify it.
  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
    return true;
  // If the operation can be done in a smaller type, do so.
  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
    return true;

  // Output known-0 bits are only known if clear in both the LHS & RHS.
  Known.Zero &= Known2.Zero;
  // Output known-1 are known to be set if set in either the LHS | RHS.
  Known.One |= Known2.One;
  break;
}
case ISD::XOR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1138, __PRETTY_FUNCTION__));
  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
                           Depth + 1))
    return true;
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1142, __PRETTY_FUNCTION__));

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If all of the demanded bits are known zero on one side, return the other.
  // These bits cannot contribute to the result of the 'xor'.
  if (DemandedBits.isSubsetOf(Known.Zero))
    return TLO.CombineTo(Op, Op0);
  if (DemandedBits.isSubsetOf(Known2.Zero))
    return TLO.CombineTo(Op, Op1);
  // If the operation can be done in a smaller type, do so.
  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
    return true;

  // If all of the unknown bits are known to be zero on one side or the other
  // (but not both) turn this into an *inclusive* or.
  //    e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));

  // Output known-0 bits are known if clear or set in both the LHS & RHS.
  KnownOut.Zero = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
  // Output known-1 are known to be set if set in only one of the LHS, RHS.
  KnownOut.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);

  if (ConstantSDNode *C = isConstOrConstSplat(Op1)) {
    // If one side is a constant, and all of the known set bits on the other
    // side are also set in the constant, turn this into an AND, as we know
    // the bits will be cleared.
    //    e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
    // NB: it is okay if more bits are known than are requested
    if (C->getAPIntValue() == Known2.One) {
      SDValue ANDC =
          TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
    }

    // If the RHS is a constant, see if we can change it. Don't alter a -1
    // constant because that's a 'not' op, and that is better for combining
    // and codegen.
    if (!C->isAllOnesValue()) {
      if (DemandedBits.isSubsetOf(C->getAPIntValue())) {
        // We're flipping all demanded bits. Flip the undemanded bits too.
        SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
        return TLO.CombineTo(Op, New);
      }
      // If we can't turn this into a 'not', try to shrink the constant.
      if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
        return true;
    }
  }

  Known = std::move(KnownOut);
  break;
}
case ISD::SELECT:
  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
                           Depth + 1))
    return true;
  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1216, __PRETTY_FUNCTION__));
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1217, __PRETTY_FUNCTION__));

  // If the operands are constants, see if we can simplify them.
  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
    return true;

  // Only known if known in both the LHS and RHS.
  Known.One &= Known2.One;
  Known.Zero &= Known2.Zero;
  break;
case ISD::SELECT_CC:
  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
                           Depth + 1))
    return true;
  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1234, __PRETTY_FUNCTION__));
  assert(!Known2.hasConflict() && "Bits known to be one AND zero?")((!Known2.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known2.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1235, __PRETTY_FUNCTION__));

  // If the operands are constants, see if we can simplify them.
  if (ShrinkDemandedConstant(Op, DemandedBits, TLO))
    return true;

  // Only known if known in both the LHS and RHS.
  Known.One &= Known2.One;
  Known.Zero &= Known2.Zero;
  break;
case ISD::SETCC: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
  // If (1) we only need the sign-bit, (2) the setcc operands are the same
  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
  // -1, we may be able to bypass the setcc.
  if (DemandedBits.isSignMask() &&
      Op0.getScalarValueSizeInBits() == BitWidth &&
      getBooleanContents(VT) ==
          BooleanContent::ZeroOrNegativeOneBooleanContent) {
    // If we're testing X < 0, then this compare isn't needed - just use X!
    // FIXME: We're limiting to integer types here, but this should also work
    // if we don't care about FP signed-zero. The use of SETLT with FP means
    // that we don't care about NaNs.
    if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
        (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
      return TLO.CombineTo(Op, Op0);

    // TODO: Should we check for other forms of sign-bit comparisons?
    // Examples: X <= -1, X >= 0
  }
  if (getBooleanContents(Op0.getValueType()) ==
          TargetLowering::ZeroOrOneBooleanContent &&
      BitWidth > 1)
    Known.Zero.setBitsFrom(1);
  break;
}
case ISD::SHL: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
    // If the shift count is an invalid immediate, don't do anything.
    if (SA->getAPIntValue().uge(BitWidth))
      break;

    unsigned ShAmt = SA->getZExtValue();
    if (ShAmt == 0)
      return TLO.CombineTo(Op, Op0);

    // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
    // single shift.  We can do this if the bottom bits (which are shifted
    // out) are never demanded.
    // TODO - support non-uniform vector amounts.
    if (Op0.getOpcode() == ISD::SRL) {
      if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
        if (ConstantSDNode *SA2 =
                isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
          if (SA2->getAPIntValue().ult(BitWidth)) {
            unsigned C1 = SA2->getZExtValue();
            unsigned Opc = ISD::SHL;
            int Diff = ShAmt - C1;
            if (Diff < 0) {
              Diff = -Diff;
              Opc = ISD::SRL;
            }

            SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType());
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
          }
        }
      }
    }

    if (SimplifyDemandedBits(Op0, DemandedBits.lshr(ShAmt), DemandedElts,
                             Known, TLO, Depth + 1))
      return true;

    // Try shrinking the operation as long as the shift amount will still be
    // in range.
    if ((ShAmt < DemandedBits.getActiveBits()) &&
        ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
      return true;

    // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
    // are not demanded. This will likely allow the anyext to be folded away.
    if (Op0.getOpcode() == ISD::ANY_EXTEND) {
      SDValue InnerOp = Op0.getOperand(0);
      EVT InnerVT = InnerOp.getValueType();
      unsigned InnerBits = InnerVT.getScalarSizeInBits();
      if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
          isTypeDesirableForOp(ISD::SHL, InnerVT)) {
        EVT ShTy = getShiftAmountTy(InnerVT, DL);
        if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
          ShTy = InnerVT;
        SDValue NarrowShl =
            TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
                            TLO.DAG.getConstant(ShAmt, dl, ShTy));
        return TLO.CombineTo(
            Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
      }
      // Repeat the SHL optimization above in cases where an extension
      // intervenes: (shl (anyext (shr x, c1)), c2) to
      // (shl (anyext x), c2-c1).  This requires that the bottom c1 bits
      // aren't demanded (as above) and that the shifted upper c1 bits of
      // x aren't demanded.
      if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
          InnerOp.hasOneUse()) {
        if (ConstantSDNode *SA2 =
                isConstOrConstSplat(InnerOp.getOperand(1))) {
          unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
          if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
              DemandedBits.getActiveBits() <=
                  (InnerBits - InnerShAmt + ShAmt) &&
              DemandedBits.countTrailingZeros() >= ShAmt) {
            SDValue NewSA = TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
                                                Op1.getValueType());
            SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
                                             InnerOp.getOperand(0));
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
          }
        }
      }
    }

    Known.Zero <<= ShAmt;
    Known.One <<= ShAmt;
    // low bits known zero.
    Known.Zero.setLowBits(ShAmt);
  }
  break;
}
case ISD::SRL: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
    // If the shift count is an invalid immediate, don't do anything.
    if (SA->getAPIntValue().uge(BitWidth))
      break;

    unsigned ShAmt = SA->getZExtValue();
    if (ShAmt == 0)
      return TLO.CombineTo(Op, Op0);

    EVT ShiftVT = Op1.getValueType();
    APInt InDemandedMask = (DemandedBits << ShAmt);

    // If the shift is exact, then it does demand the low bits (and knows that
    // they are zero).
    if (Op->getFlags().hasExact())
      InDemandedMask.setLowBits(ShAmt);

    // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
    // single shift.  We can do this if the top bits (which are shifted out)
    // are never demanded.
    // TODO - support non-uniform vector amounts.
    if (Op0.getOpcode() == ISD::SHL) {
      if (ConstantSDNode *SA2 =
              isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
        if (!DemandedBits.intersects(
                APInt::getHighBitsSet(BitWidth, ShAmt))) {
          if (SA2->getAPIntValue().ult(BitWidth)) {
            unsigned C1 = SA2->getZExtValue();
            unsigned Opc = ISD::SRL;
            int Diff = ShAmt - C1;
            if (Diff < 0) {
              Diff = -Diff;
              Opc = ISD::SHL;
            }

            SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
            return TLO.CombineTo(
                Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
          }
        }
      }
    }

    // Compute the new bits that are at the top now.
    if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1421, __PRETTY_FUNCTION__));
    Known.Zero.lshrInPlace(ShAmt);
    Known.One.lshrInPlace(ShAmt);

    Known.Zero.setHighBits(ShAmt); // High bits known zero.
  }
  break;
}
case ISD::SRA: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);

  // If this is an arithmetic shift right and only the low-bit is set, we can
  // always convert this into a logical shr, even if the shift amount is
  // variable.  The low bit of the shift cannot be an input sign bit unless
  // the shift amount is >= the size of the datatype, which is undefined.
  if (DemandedBits.isOneValue())
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));

  if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
    // If the shift count is an invalid immediate, don't do anything.
    if (SA->getAPIntValue().uge(BitWidth))
      break;

    unsigned ShAmt = SA->getZExtValue();
    if (ShAmt == 0)
      return TLO.CombineTo(Op, Op0);

    APInt InDemandedMask = (DemandedBits << ShAmt);

    // If the shift is exact, then it does demand the low bits (and knows that
    // they are zero).
    if (Op->getFlags().hasExact())
      InDemandedMask.setLowBits(ShAmt);

    // If any of the demanded bits are produced by the sign extension, we also
    // demand the input sign bit.
    if (DemandedBits.countLeadingZeros() < ShAmt)
      InDemandedMask.setSignBit();

    if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;
    assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1464, __PRETTY_FUNCTION__));
    Known.Zero.lshrInPlace(ShAmt);
    Known.One.lshrInPlace(ShAmt);

    // If the input sign bit is known to be zero, or if none of the top bits
    // are demanded, turn this into an unsigned shift right.
    if (Known.Zero[BitWidth - ShAmt - 1] ||
        DemandedBits.countLeadingZeros() >= ShAmt) {
      SDNodeFlags Flags;
      Flags.setExact(Op->getFlags().hasExact());
      return TLO.CombineTo(
          Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
    }

    int Log2 = DemandedBits.exactLogBase2();
    if (Log2 >= 0) {
      // The bit must come from the sign.
      SDValue NewSA =
          TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, Op1.getValueType());
      return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
    }

    if (Known.One[BitWidth - ShAmt - 1])
      // New bits are known one.
      Known.One.setHighBits(ShAmt);
  }
  break;
}
case ISD::FSHL:
case ISD::FSHR: {
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  SDValue Op2 = Op.getOperand(2);
  bool IsFSHL = (Op.getOpcode() == ISD::FSHL);

  if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
    unsigned Amt = SA->getAPIntValue().urem(BitWidth);

    // For fshl, 0-shift returns the 1st arg.
    // For fshr, 0-shift returns the 2nd arg.
    if (Amt == 0) {
      if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
                               Known, TLO, Depth + 1))
        return true;
      break;
    }

    // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
    // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
    APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
    APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
    if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
                             Depth + 1))
      return true;
    if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
                             Depth + 1))
      return true;

    Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
    Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
    Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
    Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
    Known.One |= Known2.One;
    Known.Zero |= Known2.Zero;
  }
  break;
}
case ISD::BITREVERSE: {
  SDValue Src = Op.getOperand(0);
  APInt DemandedSrcBits = DemandedBits.reverseBits();
  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
                           Depth + 1))
    return true;
  Known.One = Known2.One.reverseBits();
  Known.Zero = Known2.Zero.reverseBits();
  break;
}
case ISD::SIGN_EXTEND_INREG: {
  SDValue Op0 = Op.getOperand(0);
  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  unsigned ExVTBits = ExVT.getScalarSizeInBits();

  // If we only care about the highest bit, don't bother shifting right.
  if (DemandedBits.isSignMask()) {
    unsigned NumSignBits = TLO.DAG.ComputeNumSignBits(Op0);
    bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
    // However if the input is already sign extended we expect the sign
    // extension to be dropped altogether later and do not simplify.
    if (!AlreadySignExtended) {
      // Compute the correct shift amount type, which must be getShiftAmountTy
      // for scalar types after legalization.
      EVT ShiftAmtTy = VT;
      if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
        ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);

      SDValue ShiftAmt =
          TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
      return TLO.CombineTo(Op,
                           TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
    }
  }

  // If none of the extended bits are demanded, eliminate the sextinreg.
  if (DemandedBits.getActiveBits() <= ExVTBits)
    return TLO.CombineTo(Op, Op0);

  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);

  // Since the sign extended bits are demanded, we know that the sign
  // bit is demanded.
  InputDemandedBits.setBit(ExVTBits - 1);

  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1578, __PRETTY_FUNCTION__));

  // If the sign bit of the input is known set or clear, then we know the
  // top bits of the result.

  // If the input sign bit is known zero, convert this into a zero extension.
  if (Known.Zero[ExVTBits - 1])
    return TLO.CombineTo(
        Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT.getScalarType()));

  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
    Known.One.setBitsFrom(ExVTBits);
    Known.Zero &= Mask;
  } else { // Input sign bit unknown
    Known.Zero &= Mask;
    Known.One &= Mask;
  }
  break;
}
case ISD::BUILD_PAIR: {
  EVT HalfVT = Op.getOperand(0).getValueType();
  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();

  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);

  KnownBits KnownLo, KnownHi;

  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
    return true;

  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
    return true;

  Known.Zero = KnownLo.Zero.zext(BitWidth) |
               KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);

  Known.One = KnownLo.One.zext(BitWidth) |
              KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
  break;
}
case ISD::ZERO_EXTEND:
case ISD::ZERO_EXTEND_VECTOR_INREG: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned InBits = SrcVT.getScalarSizeInBits();
  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
  bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;

  // If none of the top bits are demanded, convert this into an any_extend.
  if (DemandedBits.getActiveBits() <= InBits) {
    // If we only need the non-extended bits of the bottom element
    // then we can just bitcast to the result.
    if (IsVecInReg && DemandedElts == 1 &&
        VT.getSizeInBits() == SrcVT.getSizeInBits() &&
        TLO.DAG.getDataLayout().isLittleEndian())
      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

    unsigned Opc =
        IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
    if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
  }

  APInt InDemandedBits = DemandedBits.trunc(InBits);
  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1648, __PRETTY_FUNCTION__));
  assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1649, __PRETTY_FUNCTION__));
  Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
  break;
}
case ISD::SIGN_EXTEND:
case ISD::SIGN_EXTEND_VECTOR_INREG: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned InBits = SrcVT.getScalarSizeInBits();
  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
  bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;

  // If none of the top bits are demanded, convert this into an any_extend.
  if (DemandedBits.getActiveBits() <= InBits) {
    // If we only need the non-extended bits of the bottom element
    // then we can just bitcast to the result.
    if (IsVecInReg && DemandedElts == 1 &&
        VT.getSizeInBits() == SrcVT.getSizeInBits() &&
        TLO.DAG.getDataLayout().isLittleEndian())
      return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

    unsigned Opc =
        IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
    if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
  }

  APInt InDemandedBits = DemandedBits.trunc(InBits);
  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);

  // Since some of the sign extended bits are demanded, we know that the sign
  // bit is demanded.
  InDemandedBits.setBit(InBits - 1);

  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1686, __PRETTY_FUNCTION__));
  assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1687, __PRETTY_FUNCTION__));

  // If the sign bit is known one, the top bits match.
  Known = Known.sext(BitWidth);

  // If the sign bit is known zero, convert this to a zero extend.
  if (Known.isNonNegative()) {
    unsigned Opc =
        IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
    if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
      return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
  }
  break;
}
case ISD::ANY_EXTEND:
case ISD::ANY_EXTEND_VECTOR_INREG: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned InBits = SrcVT.getScalarSizeInBits();
  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
  bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;

  // If we only need the bottom element then we can just bitcast.
  // TODO: Handle ANY_EXTEND?
  if (IsVecInReg && DemandedElts == 1 &&
      VT.getSizeInBits() == SrcVT.getSizeInBits() &&
      TLO.DAG.getDataLayout().isLittleEndian())
    return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));

  APInt InDemandedBits = DemandedBits.trunc(InBits);
  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
                           Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1721, __PRETTY_FUNCTION__));
  assert(Known.getBitWidth() == InBits && "Src width has changed?")((Known.getBitWidth() == InBits && "Src width has changed?"
) ? static_cast<void> (0) : __assert_fail ("Known.getBitWidth() == InBits && \"Src width has changed?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1722, __PRETTY_FUNCTION__));
  Known = Known.zext(BitWidth, false /* => any extend */);
  break;
}
case ISD::TRUNCATE: {
  SDValue Src = Op.getOperand(0);

  // Simplify the input, using demanded bit information, and compute the known
  // zero/one bits live out.
  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
  if (SimplifyDemandedBits(Src, TruncMask, Known, TLO, Depth + 1))
    return true;
  Known = Known.trunc(BitWidth);

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
          Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
    return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));

  // If the input is only used by this truncate, see if we can shrink it based
  // on the known demanded bits.
  if (Src.getNode()->hasOneUse()) {
    switch (Src.getOpcode()) {
    default:
      break;
    case ISD::SRL:
      // Shrink SRL by a constant if none of the high bits shifted in are
      // demanded.
      if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
        // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
        // undesirable.
        break;

      auto *ShAmt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
      if (!ShAmt || ShAmt->getAPIntValue().uge(BitWidth))
        break;

      SDValue Shift = Src.getOperand(1);
      uint64_t ShVal = ShAmt->getZExtValue();

      if (TLO.LegalTypes())
        Shift = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL));

      APInt HighBits =
          APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
      HighBits.lshrInPlace(ShVal);
      HighBits = HighBits.trunc(BitWidth);

      if (!(HighBits & DemandedBits)) {
        // None of the shifted in bits are needed.  Add a truncate of the
        // shift input, then shift it.
        SDValue NewTrunc =
            TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
        return TLO.CombineTo(
            Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, Shift));
      }
      break;
    }
  }

  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1783, __PRETTY_FUNCTION__));
  break;
}
case ISD::AssertZext: {
  // AssertZext demands all of the high bits, plus any of the low bits
  // demanded by its users.
  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
                           TLO, Depth + 1))
    return true;
  assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 1794, __PRETTY_FUNCTION__));

  Known.Zero |= ~InMask;
  break;
}
case ISD::EXTRACT_VECTOR_ELT: {
  SDValue Src = Op.getOperand(0);
  SDValue Idx = Op.getOperand(1);
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  unsigned EltBitWidth = Src.getScalarValueSizeInBits();

  // Demand the bits from every vector element without a constant index.
  APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
    if (CIdx->getAPIntValue().ult(NumSrcElts))
      DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());

  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
  // anything about the extended bits.
  APInt DemandedSrcBits = DemandedBits;
  if (BitWidth > EltBitWidth)
    DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);

  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
                           Depth + 1))
    return true;

  Known = Known2;
  if (BitWidth > EltBitWidth)
    Known = Known.zext(BitWidth, false /* => any extend */);
  break;
}
case ISD::BITCAST: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();
  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();

  // If this is an FP->Int bitcast and if the sign bit is the only
  // thing demanded, turn this into a FGETSIGN.
  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
      DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
      SrcVT.isFloatingPoint()) {
    bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
    bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
    if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
        SrcVT != MVT::f128) {
      // Cannot eliminate/lower SHL for f128 yet.
      EVT Ty = OpVTLegal ? VT : MVT::i32;
      // Make a FGETSIGN + SHL to move the sign bit into the appropriate
      // place.  We expect the SHL to be eliminated by other optimizations.
      SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
      unsigned OpVTSizeInBits = Op.getValueSizeInBits();
      if (!OpVTLegal && OpVTSizeInBits > 32)
        Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
      unsigned ShVal = Op.getValueSizeInBits() - 1;
      SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
      return TLO.CombineTo(Op,
                           TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
    }
  }

  // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
  // Demand the elt/bit if any of the original elts/bits are demanded.
  // TODO - bigendian once we have test coverage.
  if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
      TLO.DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = BitWidth / NumSrcEltBits;
    unsigned NumSrcElts = SrcVT.getVectorNumElements();
    APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
    for (unsigned i = 0; i != Scale; ++i) {
      unsigned Offset = i * NumSrcEltBits;
      APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
      if (!Sub.isNullValue()) {
        DemandedSrcBits |= Sub;
        for (unsigned j = 0; j != NumElts; ++j)
          if (DemandedElts[j])
            DemandedSrcElts.setBit((j * Scale) + i);
      }
    }

    APInt KnownSrcUndef, KnownSrcZero;
    if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
                                   KnownSrcZero, TLO, Depth + 1))
      return true;

    KnownBits KnownSrcBits;
    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
                             KnownSrcBits, TLO, Depth + 1))
      return true;
  } else if ((NumSrcEltBits % BitWidth) == 0 &&
             TLO.DAG.getDataLayout().isLittleEndian()) {
    unsigned Scale = NumSrcEltBits / BitWidth;
    unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
    APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
    APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
    for (unsigned i = 0; i != NumElts; ++i)
      if (DemandedElts[i]) {
        unsigned Offset = (i % Scale) * BitWidth;
        DemandedSrcBits.insertBits(DemandedBits, Offset);
        DemandedSrcElts.setBit(i / Scale);
      }

    if (SrcVT.isVector()) {
      APInt KnownSrcUndef, KnownSrcZero;
      if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
                                     KnownSrcZero, TLO, Depth + 1))
        return true;
    }

    KnownBits KnownSrcBits;
    if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
                             KnownSrcBits, TLO, Depth + 1))
      return true;
  }

  // If this is a bitcast, let computeKnownBits handle it.  Only do this on a
  // recursive call where Known may be useful to the caller.
  if (Depth > 0) {
    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
    return false;
  }
  break;
}
case ISD::ADD:
case ISD::MUL:
case ISD::SUB: {
  // Add, Sub, and Mul don't demand any bits in positions beyond that
  // of the highest bit demanded of them.
  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
  SDNodeFlags Flags = Op.getNode()->getFlags();
  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
                           Depth + 1) ||
      SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
                           Depth + 1) ||
      // See if the operation should be performed at a smaller bit width.
      ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
    if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
      // Disable the nsw and nuw flags. We can no longer guarantee that we
      // won't wrap after simplification.
      Flags.setNoSignedWrap(false);
      Flags.setNoUnsignedWrap(false);
      SDValue NewOp =
          TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
      return TLO.CombineTo(Op, NewOp);
    }
    return true;
  }

  // Attempt to avoid multi-use ops if we don't need anything from them.
  if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
    SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
        Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
    SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
        Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
    if (DemandedOp0 || DemandedOp1) {
      Flags.setNoSignedWrap(false);
      Flags.setNoUnsignedWrap(false);
      Op0 = DemandedOp0 ? DemandedOp0 : Op0;
      Op1 = DemandedOp1 ? DemandedOp1 : Op1;
      SDValue NewOp =
          TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
      return TLO.CombineTo(Op, NewOp);
    }
  }

  // If we have a constant operand, we may be able to turn it into -1 if we
  // do not demand the high bits. This can make the constant smaller to
  // encode, allow more general folding, or match specialized instruction
  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
  // is probably not useful (and could be detrimental).
  ConstantSDNode *C = isConstOrConstSplat(Op1);
  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
  if (C && !C->isAllOnesValue() && !C->isOne() &&
      (C->getAPIntValue() | HighMask).isAllOnesValue()) {
    SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
    // Disable the nsw and nuw flags. We can no longer guarantee that we
    // won't wrap after simplification.
    Flags.setNoSignedWrap(false);
    Flags.setNoUnsignedWrap(false);
    SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
    return TLO.CombineTo(Op, NewOp);
  }

  LLVM_FALLTHROUGH[[gnu::fallthrough]];
}
default:
  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
    if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
                                          Known, TLO, Depth))
      return true;
    break;
  }

  // Just use computeKnownBits to compute output bits.
  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
  break;
}

// If we know the value of all of the demanded bits, return this as a
// constant.
if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
  // Avoid folding to a constant if any OpaqueConstant is involved.
  const SDNode *N = Op.getNode();
  for (SDNodeIterator I = SDNodeIterator::begin(N),
                      E = SDNodeIterator::end(N);
       I != E; ++I) {
    SDNode *Op = *I;
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
      if (C->isOpaque())
        return false;
  }
  // TODO: Handle float bits as well.
  if (VT.isInteger())
    return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
}

return false;
2014}

2016bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
                                              const APInt &DemandedElts,
                                              APInt &KnownUndef,
                                              APInt &KnownZero,
                                              DAGCombinerInfo &DCI) const {
SelectionDAG &DAG = DCI.DAG;
TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                      !DCI.isBeforeLegalizeOps());

bool Simplified =
    SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
if (Simplified) {
  DCI.AddToWorklist(Op.getNode());
  DCI.CommitTargetLoweringOpt(TLO);
}

return Simplified;
2033}

2035/// Given a vector binary operation and known undefined elements for each input
2036/// operand, compute whether each element of the output is undefined.
2037static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
                                       const APInt &UndefOp0,
                                       const APInt &UndefOp1) {
EVT VT = BO.getValueType();
assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
 VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2042, __PRETTY_FUNCTION__))
       "Vector binop only")((DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) &&
 VT.isVector() && "Vector binop only") ? static_cast<
void> (0) : __assert_fail ("DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && \"Vector binop only\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2042, __PRETTY_FUNCTION__));

EVT EltVT = VT.getVectorElementType();
unsigned NumElts = VT.getVectorNumElements();
assert(UndefOp0.getBitWidth() == NumElts &&((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2047, __PRETTY_FUNCTION__))
       UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis")((UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth
() == NumElts && "Bad type for undef analysis") ? static_cast
<void> (0) : __assert_fail ("UndefOp0.getBitWidth() == NumElts && UndefOp1.getBitWidth() == NumElts && \"Bad type for undef analysis\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2047, __PRETTY_FUNCTION__));

auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
                                 const APInt &UndefVals) {
  if (UndefVals[Index])
    return DAG.getUNDEF(EltVT);

  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
    // Try hard to make sure that the getNode() call is not creating temporary
    // nodes. Ignore opaque integers because they do not constant fold.
    SDValue Elt = BV->getOperand(Index);
    auto *C = dyn_cast<ConstantSDNode>(Elt);
    if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
      return Elt;
  }

  return SDValue();
};

APInt KnownUndef = APInt::getNullValue(NumElts);
for (unsigned i = 0; i != NumElts; ++i) {
  // If both inputs for this element are either constant or undef and match
  // the element type, compute the constant/undef result for this element of
  // the vector.
  // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
  // not handle FP constants. The code within getNode() should be refactored
  // to avoid the danger of creating a bogus temporary node here.
  SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
  SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
  if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
    if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
      KnownUndef.setBit(i);
}
return KnownUndef;
2081}

2083bool TargetLowering::SimplifyDemandedVectorElts(
  SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
  bool AssumeSingleUse) const {
EVT VT = Op.getValueType();
APInt DemandedElts = OriginalDemandedElts;
unsigned NumElts = DemandedElts.getBitWidth();
assert(VT.isVector() && "Expected vector op")((VT.isVector() && "Expected vector op") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Expected vector op\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2090, __PRETTY_FUNCTION__));
assert(VT.getVectorNumElements() == NumElts &&((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2092, __PRETTY_FUNCTION__))
       "Mask size mismatches value type element count!")((VT.getVectorNumElements() == NumElts && "Mask size mismatches value type element count!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == NumElts && \"Mask size mismatches value type element count!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2092, __PRETTY_FUNCTION__));

KnownUndef = KnownZero = APInt::getNullValue(NumElts);

// Undef operand.
if (Op.isUndef()) {
  KnownUndef.setAllBits();
  return false;
}

// If Op has other users, assume that all elements are needed.
if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
  DemandedElts.setAllBits();

// Not demanding any elements from Op.
if (DemandedElts == 0) {
  KnownUndef.setAllBits();
  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
}

// Limit search depth.
if (Depth >= SelectionDAG::MaxRecursionDepth)
  return false;

SDLoc DL(Op);
unsigned EltSizeInBits = VT.getScalarSizeInBits();

switch (Op.getOpcode()) {
case ISD::SCALAR_TO_VECTOR: {
  if (!DemandedElts[0]) {
    KnownUndef.setAllBits();
    return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
  }
  KnownUndef.setHighBits(NumElts - 1);
  break;
}
case ISD::BITCAST: {
  SDValue Src = Op.getOperand(0);
  EVT SrcVT = Src.getValueType();

  // We only handle vectors here.
  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
  if (!SrcVT.isVector())
    break;

  // Fast handling of 'identity' bitcasts.
  unsigned NumSrcElts = SrcVT.getVectorNumElements();
  if (NumSrcElts == NumElts)
    return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
                                      KnownZero, TLO, Depth + 1);

  APInt SrcZero, SrcUndef;
  APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);

  // Bitcast from 'large element' src vector to 'small element' vector, we
  // must demand a source element if any DemandedElt maps to it.
  if ((NumElts % NumSrcElts) == 0) {
    unsigned Scale = NumElts / NumSrcElts;
    for (unsigned i = 0; i != NumElts; ++i)
      if (DemandedElts[i])
        SrcDemandedElts.setBit(i / Scale);

    if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                   TLO, Depth + 1))
      return true;

    // Try calling SimplifyDemandedBits, converting demanded elts to the bits
    // of the large element.
    // TODO - bigendian once we have test coverage.
    if (TLO.DAG.getDataLayout().isLittleEndian()) {
      unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
      APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
      for (unsigned i = 0; i != NumElts; ++i)
        if (DemandedElts[i]) {
          unsigned Ofs = (i % Scale) * EltSizeInBits;
          SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
        }

      KnownBits Known;
      if (SimplifyDemandedBits(Src, SrcDemandedBits, Known, TLO, Depth + 1))
        return true;
    }

    // If the src element is zero/undef then all the output elements will be -
    // only demanded elements are guaranteed to be correct.
    for (unsigned i = 0; i != NumSrcElts; ++i) {
      if (SrcDemandedElts[i]) {
        if (SrcZero[i])
          KnownZero.setBits(i * Scale, (i + 1) * Scale);
        if (SrcUndef[i])
          KnownUndef.setBits(i * Scale, (i + 1) * Scale);
      }
    }
  }

  // Bitcast from 'small element' src vector to 'large element' vector, we
  // demand all smaller source elements covered by the larger demanded element
  // of this vector.
  if ((NumSrcElts % NumElts) == 0) {
    unsigned Scale = NumSrcElts / NumElts;
    for (unsigned i = 0; i != NumElts; ++i)
      if (DemandedElts[i])
        SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);

    if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
                                   TLO, Depth + 1))
      return true;

    // If all the src elements covering an output element are zero/undef, then
    // the output element will be as well, assuming it was demanded.
    for (unsigned i = 0; i != NumElts; ++i) {
      if (DemandedElts[i]) {
        if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
          KnownZero.setBit(i);
        if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
          KnownUndef.setBit(i);
      }
    }
  }
  break;
}
case ISD::BUILD_VECTOR: {
  // Check all elements and simplify any unused elements with UNDEF.
  if (!DemandedElts.isAllOnesValue()) {
    // Don't simplify BROADCASTS.
    if (llvm::any_of(Op->op_values(),
                     [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
      SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
      bool Updated = false;
      for (unsigned i = 0; i != NumElts; ++i) {
        if (!DemandedElts[i] && !Ops[i].isUndef()) {
          Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
          KnownUndef.setBit(i);
          Updated = true;
        }
      }
      if (Updated)
        return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
    }
  }
  for (unsigned i = 0; i != NumElts; ++i) {
    SDValue SrcOp = Op.getOperand(i);
    if (SrcOp.isUndef()) {
      KnownUndef.setBit(i);
    } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
               (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
      KnownZero.setBit(i);
    }
  }
  break;
}
case ISD::CONCAT_VECTORS: {
  EVT SubVT = Op.getOperand(0).getValueType();
  unsigned NumSubVecs = Op.getNumOperands();
  unsigned NumSubElts = SubVT.getVectorNumElements();
  for (unsigned i = 0; i != NumSubVecs; ++i) {
    SDValue SubOp = Op.getOperand(i);
    APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
    APInt SubUndef, SubZero;
    if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
                                   Depth + 1))
      return true;
    KnownUndef.insertBits(SubUndef, i * NumSubElts);
    KnownZero.insertBits(SubZero, i * NumSubElts);
  }
  break;
}
case ISD::INSERT_SUBVECTOR: {
  if (!isa<ConstantSDNode>(Op.getOperand(2)))
    break;
  SDValue Base = Op.getOperand(0);
  SDValue Sub = Op.getOperand(1);
  EVT SubVT = Sub.getValueType();
  unsigned NumSubElts = SubVT.getVectorNumElements();
  const APInt &Idx = Op.getConstantOperandAPInt(2);
  if (Idx.ugt(NumElts - NumSubElts))
    break;
  unsigned SubIdx = Idx.getZExtValue();
  APInt SubElts = DemandedElts.extractBits(NumSubElts, SubIdx);
  APInt SubUndef, SubZero;
  if (SimplifyDemandedVectorElts(Sub, SubElts, SubUndef, SubZero, TLO,
                                 Depth + 1))
    return true;
  APInt BaseElts = DemandedElts;
  BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx);

  // If none of the base operand elements are demanded, replace it with undef.
  if (!BaseElts && !Base.isUndef())
    return TLO.CombineTo(Op,
                         TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
                                         TLO.DAG.getUNDEF(VT),
                                         Op.getOperand(1),
                                         Op.getOperand(2)));

  if (SimplifyDemandedVectorElts(Base, BaseElts, KnownUndef, KnownZero, TLO,
                                 Depth + 1))
    return true;
  KnownUndef.insertBits(SubUndef, SubIdx);
  KnownZero.insertBits(SubZero, SubIdx);
  break;
}
case ISD::EXTRACT_SUBVECTOR: {
  SDValue Src = Op.getOperand(0);
  ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
    // Offset the demanded elts by the subvector index.
    uint64_t Idx = SubIdx->getZExtValue();
    APInt SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
    APInt SrcUndef, SrcZero;
    if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
                                   Depth + 1))
      return true;
    KnownUndef = SrcUndef.extractBits(NumElts, Idx);
    KnownZero = SrcZero.extractBits(NumElts, Idx);
  }
  break;
}
case ISD::INSERT_VECTOR_ELT: {
  SDValue Vec = Op.getOperand(0);
  SDValue Scl = Op.getOperand(1);
  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));

  // For a legal, constant insertion index, if we don't need this insertion
  // then strip it, else remove it from the demanded elts.
  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
    unsigned Idx = CIdx->getZExtValue();
    if (!DemandedElts[Idx])
      return TLO.CombineTo(Op, Vec);

    APInt DemandedVecElts(DemandedElts);
    DemandedVecElts.clearBit(Idx);
    if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
                                   KnownZero, TLO, Depth + 1))
      return true;

    KnownUndef.clearBit(Idx);
    if (Scl.isUndef())
      KnownUndef.setBit(Idx);

    KnownZero.clearBit(Idx);
    if (isNullConstant(Scl) || isNullFPConstant(Scl))
      KnownZero.setBit(Idx);
    break;
  }

  APInt VecUndef, VecZero;
  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
                                 Depth + 1))
    return true;
  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
  break;
}
case ISD::VSELECT: {
  // Try to transform the select condition based on the current demanded
  // elements.
  // TODO: If a condition element is undef, we can choose from one arm of the
  //       select (and if one arm is undef, then we can propagate that to the
  //       result).
  // TODO - add support for constant vselect masks (see IR version of this).
  APInt UnusedUndef, UnusedZero;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
                                 UnusedZero, TLO, Depth + 1))
    return true;

  // See if we can simplify either vselect operand.
  APInt DemandedLHS(DemandedElts);
  APInt DemandedRHS(DemandedElts);
  APInt UndefLHS, ZeroLHS;
  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
                                 ZeroLHS, TLO, Depth + 1))
    return true;
  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
                                 ZeroRHS, TLO, Depth + 1))
    return true;

  KnownUndef = UndefLHS & UndefRHS;
  KnownZero = ZeroLHS & ZeroRHS;
  break;
}
case ISD::VECTOR_SHUFFLE: {
  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();

  // Collect demanded elements from shuffle operands..
  APInt DemandedLHS(NumElts, 0);
  APInt DemandedRHS(NumElts, 0);
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = ShuffleMask[i];
    if (M < 0 || !DemandedElts[i])
      continue;
    assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range")((0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < (int)(2 * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2383, __PRETTY_FUNCTION__));
    if (M < (int)NumElts)
      DemandedLHS.setBit(M);
    else
      DemandedRHS.setBit(M - NumElts);
  }

  // See if we can simplify either shuffle operand.
  APInt UndefLHS, ZeroLHS;
  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
                                 ZeroLHS, TLO, Depth + 1))
    return true;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
                                 ZeroRHS, TLO, Depth + 1))
    return true;

  // Simplify mask using undef elements from LHS/RHS.
  bool Updated = false;
  bool IdentityLHS = true, IdentityRHS = true;
  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
  for (unsigned i = 0; i != NumElts; ++i) {
    int &M = NewMask[i];
    if (M < 0)
      continue;
    if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
        (M >= (int)NumElts && UndefRHS[M - NumElts])) {
      Updated = true;
      M = -1;
    }
    IdentityLHS &= (M < 0) || (M == (int)i);
    IdentityRHS &= (M < 0) || ((M - NumElts) == i);
  }

  // Update legal shuffle masks based on demanded elements if it won't reduce
  // to Identity which can cause premature removal of the shuffle mask.
  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
    SDValue LegalShuffle =
        buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
                                NewMask, TLO.DAG);
    if (LegalShuffle)
      return TLO.CombineTo(Op, LegalShuffle);
  }

  // Propagate undef/zero elements from LHS/RHS.
  for (unsigned i = 0; i != NumElts; ++i) {
    int M = ShuffleMask[i];
    if (M < 0) {
      KnownUndef.setBit(i);
    } else if (M < (int)NumElts) {
      if (UndefLHS[M])
        KnownUndef.setBit(i);
      if (ZeroLHS[M])
        KnownZero.setBit(i);
    } else {
      if (UndefRHS[M - NumElts])
        KnownUndef.setBit(i);
      if (ZeroRHS[M - NumElts])
        KnownZero.setBit(i);
    }
  }
  break;
}
case ISD::ANY_EXTEND_VECTOR_INREG:
case ISD::SIGN_EXTEND_VECTOR_INREG:
case ISD::ZERO_EXTEND_VECTOR_INREG: {
  APInt SrcUndef, SrcZero;
  SDValue Src = Op.getOperand(0);
  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
                                 Depth + 1))
    return true;
  KnownZero = SrcZero.zextOrTrunc(NumElts);
  KnownUndef = SrcUndef.zextOrTrunc(NumElts);

  if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
      Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
      DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
    // aext - if we just need the bottom element then we can bitcast.
    return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
  }

  if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
    // zext(undef) upper bits are guaranteed to be zero.
    if (DemandedElts.isSubsetOf(KnownUndef))
      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
    KnownUndef.clearAllBits();
  }
  break;
}

// TODO: There are more binop opcodes that could be handled here - MUL, MIN,
// MAX, saturated math, etc.
case ISD::OR:
case ISD::XOR:
case ISD::ADD:
case ISD::SUB:
case ISD::FADD:
case ISD::FSUB:
case ISD::FMUL:
case ISD::FDIV:
case ISD::FREM: {
  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
                                 ZeroRHS, TLO, Depth + 1))
    return true;
  APInt UndefLHS, ZeroLHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
                                 ZeroLHS, TLO, Depth + 1))
    return true;

  KnownZero = ZeroLHS & ZeroRHS;
  KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
  break;
}
case ISD::SHL:
case ISD::SRL:
case ISD::SRA:
case ISD::ROTL:
case ISD::ROTR: {
  APInt UndefRHS, ZeroRHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
                                 ZeroRHS, TLO, Depth + 1))
    return true;
  APInt UndefLHS, ZeroLHS;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
                                 ZeroLHS, TLO, Depth + 1))
    return true;

  KnownZero = ZeroLHS;
  KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
  break;
}
case ISD::MUL:
case ISD::AND: {
  APInt SrcUndef, SrcZero;
  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
                                 SrcZero, TLO, Depth + 1))
    return true;
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
                                 KnownZero, TLO, Depth + 1))
    return true;

  // If either side has a zero element, then the result element is zero, even
  // if the other is an UNDEF.
  // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
  // and then handle 'and' nodes with the rest of the binop opcodes.
  KnownZero |= SrcZero;
  KnownUndef &= SrcUndef;
  KnownUndef &= ~KnownZero;
  break;
}
case ISD::TRUNCATE:
case ISD::SIGN_EXTEND:
case ISD::ZERO_EXTEND:
  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
                                 KnownZero, TLO, Depth + 1))
    return true;

  if (Op.getOpcode() == ISD::ZERO_EXTEND) {
    // zext(undef) upper bits are guaranteed to be zero.
    if (DemandedElts.isSubsetOf(KnownUndef))
      return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
    KnownUndef.clearAllBits();
  }
  break;
default: {
  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
    if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
                                                KnownZero, TLO, Depth))
      return true;
  } else {
    KnownBits Known;
    APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
    if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
                             TLO, Depth, AssumeSingleUse))
      return true;
  }
  break;
}
}
assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero")(((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero"
) ? static_cast<void> (0) : __assert_fail ("(KnownUndef & KnownZero) == 0 && \"Elements flagged as undef AND zero\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2565, __PRETTY_FUNCTION__));

// Constant fold all undef cases.
// TODO: Handle zero cases as well.
if (DemandedElts.isSubsetOf(KnownUndef))
  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));

return false;
2573}

2575/// Determine which of the bits specified in Mask are known to be either zero or
2576/// one and return them in the Known.
2577void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
                                                 KnownBits &Known,
                                                 const APInt &DemandedElts,
                                                 const SelectionDAG &DAG,
                                                 unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2587, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2587, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2587, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2587, __PRETTY_FUNCTION__))
       "Should use MaskedValueIsZero if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2587, __PRETTY_FUNCTION__))
       " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use MaskedValueIsZero if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2587, __PRETTY_FUNCTION__));
Known.resetAll();
2589}

2591void TargetLowering::computeKnownBitsForTargetInstr(
  GISelKnownBits &Analysis, Register R, KnownBits &Known,
  const APInt &DemandedElts, const MachineRegisterInfo &MRI,
  unsigned Depth) const {
Known.resetAll();
2596}

2598void TargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
                                                 KnownBits &Known,
                                                 const APInt &DemandedElts,
                                                 const SelectionDAG &DAG,
                                                 unsigned Depth) const {
assert(isa<FrameIndexSDNode>(Op) && "expected FrameIndex")((isa<FrameIndexSDNode>(Op) && "expected FrameIndex"
) ? static_cast<void> (0) : __assert_fail ("isa<FrameIndexSDNode>(Op) && \"expected FrameIndex\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2603, __PRETTY_FUNCTION__));

if (unsigned Align = DAG.InferPtrAlignment(Op)) {
  // The low bits are known zero if the pointer is aligned.
  Known.Zero.setLowBits(Log2_32(Align));
}
2609}

2611/// This method can be implemented by targets that want to expose additional
2612/// information about sign bits to the DAG Combiner.
2613unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
                                                       const APInt &,
                                                       const SelectionDAG &,
                                                       unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2622, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2622, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2622, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2622, __PRETTY_FUNCTION__))
       "Should use ComputeNumSignBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2622, __PRETTY_FUNCTION__))
       " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use ComputeNumSignBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use ComputeNumSignBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2622, __PRETTY_FUNCTION__));
return 1;
2624}

2626bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
  TargetLoweringOpt &TLO, unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2634, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2634, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2634, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2634, __PRETTY_FUNCTION__))
       "Should use SimplifyDemandedVectorElts if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2634, __PRETTY_FUNCTION__))
       " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedVectorElts if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedVectorElts if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2634, __PRETTY_FUNCTION__));
return false;
2636}

2638bool TargetLowering::SimplifyDemandedBitsForTargetNode(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2646, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2646, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2646, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2646, __PRETTY_FUNCTION__))
       "Should use SimplifyDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2646, __PRETTY_FUNCTION__))
       " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2646, __PRETTY_FUNCTION__));
computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
return false;
2649}

2651SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
  SelectionDAG &DAG, unsigned Depth) const {
assert((((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__))
    (Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__))
     Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__))
    "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__))
    " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use SimplifyMultipleUseDemandedBits if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2660, __PRETTY_FUNCTION__));
return SDValue();
2662}

2664SDValue
2665TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
                                      SDValue N1, MutableArrayRef<int> Mask,
                                      SelectionDAG &DAG) const {
bool LegalMask = isShuffleMaskLegal(Mask, VT);
if (!LegalMask) {
  std::swap(N0, N1);
  ShuffleVectorSDNode::commuteMask(Mask);
  LegalMask = isShuffleMaskLegal(Mask, VT);
}

if (!LegalMask)
  return SDValue();

return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
2679}

2681const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
return nullptr;
2683}

2685bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
                                                const SelectionDAG &DAG,
                                                bool SNaN,
                                                unsigned Depth) const {
assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__))
        Op.getOpcode() == ISD::INTRINSIC_VOID) &&(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__))
       "Should use isKnownNeverNaN if you don't know whether Op"(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__))
       " is a target node!")(((Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode()
 == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN
 || Op.getOpcode() == ISD::INTRINSIC_VOID) && "Should use isKnownNeverNaN if you don't know whether Op"
 " is a target node!") ? static_cast<void> (0) : __assert_fail
 ("(Op.getOpcode() >= ISD::BUILTIN_OP_END || Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || Op.getOpcode() == ISD::INTRINSIC_VOID) && \"Should use isKnownNeverNaN if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2694, __PRETTY_FUNCTION__));
return false;
2696}

2698// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
2699// work with truncating build vectors and vectors with elements of less than
2700// 8 bits.
2701bool TargetLowering::isConstTrueVal(const SDNode *N) const {
if (!N)
  return false;

APInt CVal;
if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
  CVal = CN->getAPIntValue();
} else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
  auto *CN = BV->getConstantSplatNode();
  if (!CN)
    return false;

  // If this is a truncating build vector, truncate the splat value.
  // Otherwise, we may fail to match the expected values below.
  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
  CVal = CN->getAPIntValue();
  if (BVEltWidth < CVal.getBitWidth())
    CVal = CVal.trunc(BVEltWidth);
} else {
  return false;
}

switch (getBooleanContents(N->getValueType(0))) {
case UndefinedBooleanContent:
  return CVal[0];
case ZeroOrOneBooleanContent:
  return CVal.isOneValue();
case ZeroOrNegativeOneBooleanContent:
  return CVal.isAllOnesValue();
}

llvm_unreachable("Invalid boolean contents")::llvm::llvm_unreachable_internal("Invalid boolean contents",
 "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2732);
2733}

2735bool TargetLowering::isConstFalseVal(const SDNode *N) const {
if (!N)
  return false;

const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
if (!CN) {
  const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
  if (!BV)
    return false;

  // Only interested in constant splats, we don't care about undef
  // elements in identifying boolean constants and getConstantSplatNode
  // returns NULL if all ops are undef;
  CN = BV->getConstantSplatNode();
  if (!CN)
    return false;
}

if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
  return !CN->getAPIntValue()[0];

return CN->isNullValue();
2757}

2759bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
                                     bool SExt) const {
if (VT == MVT::i1)
  return N->isOne();

TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
switch (Cnt) {
case TargetLowering::ZeroOrOneBooleanContent:
  // An extended value of 1 is always true, unless its original type is i1,
  // in which case it will be sign extended to -1.
  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
case TargetLowering::UndefinedBooleanContent:
case TargetLowering::ZeroOrNegativeOneBooleanContent:
  return N->isAllOnesValue() && SExt;
}
llvm_unreachable("Unexpected enumeration.")::llvm::llvm_unreachable_internal("Unexpected enumeration.", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2774);
2775}

2777/// This helper function of SimplifySetCC tries to optimize the comparison when
2778/// either operand of the SetCC node is a bitwise-and instruction.
2779SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
                                       ISD::CondCode Cond, const SDLoc &DL,
                                       DAGCombinerInfo &DCI) const {
// Match these patterns in any of their permutations:
// (X & Y) == Y
// (X & Y) != Y
if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
  std::swap(N0, N1);

EVT OpVT = N0.getValueType();
if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
    (Cond != ISD::SETEQ && Cond != ISD::SETNE))
  return SDValue();

SDValue X, Y;
if (N0.getOperand(0) == N1) {
  X = N0.getOperand(1);
  Y = N0.getOperand(0);
} else if (N0.getOperand(1) == N1) {
  X = N0.getOperand(0);
  Y = N0.getOperand(1);
} else {
  return SDValue();
}

SelectionDAG &DAG = DCI.DAG;
SDValue Zero = DAG.getConstant(0, DL, OpVT);
if (DAG.isKnownToBeAPowerOfTwo(Y)) {
  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
  // Note that where Y is variable and is known to have at most one bit set
  // (for example, if it is Z & 1) we cannot do this; the expressions are not
  // equivalent when Y == 0.
  Cond = ISD::getSetCCInverse(Cond, /*isInteger=*/true);
  if (DCI.isBeforeLegalizeOps() ||
      isCondCodeLegal(Cond, N0.getSimpleValueType()))
    return DAG.getSetCC(DL, VT, N0, Zero, Cond);
} else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
  // If the target supports an 'and-not' or 'and-complement' logic operation,
  // try to use that to make a comparison operation more efficient.
  // But don't do this transform if the mask is a single bit because there are
  // more efficient ways to deal with that case (for example, 'bt' on x86 or
  // 'rlwinm' on PPC).

  // Bail out if the compare operand that we want to turn into a zero is
  // already a zero (otherwise, infinite loop).
  auto *YConst = dyn_cast<ConstantSDNode>(Y);
  if (YConst && YConst->isNullValue())
    return SDValue();

  // Transform this into: ~X & Y == 0.
  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
}

return SDValue();
2835}

2837/// There are multiple IR patterns that could be checking whether certain
2838/// truncation of a signed number would be lossy or not. The pattern which is
2839/// best at IR level, may not lower optimally. Thus, we want to unfold it.
2840/// We are looking for the following pattern: (KeptBits is a constant)
2841///   (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
2842/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
2843/// KeptBits also can't be 1, that would have been folded to  %x dstcond 0
2844/// We will unfold it into the natural trunc+sext pattern:
2845///   ((%x << C) a>> C) dstcond %x
2846/// Where  C = bitwidth(x) - KeptBits  and  C u< bitwidth(x)
2847SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
  const SDLoc &DL) const {
// We must be comparing with a constant.
ConstantSDNode *C1;
if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
  return SDValue();

// N0 should be:  add %x, (1 << (KeptBits-1))
if (N0->getOpcode() != ISD::ADD)
  return SDValue();

// And we must be 'add'ing a constant.
ConstantSDNode *C01;
if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
  return SDValue();

SDValue X = N0->getOperand(0);
EVT XVT = X.getValueType();

// Validate constants ...

APInt I1 = C1->getAPIntValue();

ISD::CondCode NewCond;
if (Cond == ISD::CondCode::SETULT) {
  NewCond = ISD::CondCode::SETEQ;
} else if (Cond == ISD::CondCode::SETULE) {
  NewCond = ISD::CondCode::SETEQ;
  // But need to 'canonicalize' the constant.
  I1 += 1;
} else if (Cond == ISD::CondCode::SETUGT) {
  NewCond = ISD::CondCode::SETNE;
  // But need to 'canonicalize' the constant.
  I1 += 1;
} else if (Cond == ISD::CondCode::SETUGE) {
  NewCond = ISD::CondCode::SETNE;
} else
  return SDValue();

APInt I01 = C01->getAPIntValue();

auto checkConstants = [&I1, &I01]() -> bool {
  // Both of them must be power-of-two, and the constant from setcc is bigger.
  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
};

if (checkConstants()) {
  // Great, e.g. got  icmp ult i16 (add i16 %x, 128), 256
} else {
  // What if we invert constants? (and the target predicate)
  I1.negate();
  I01.negate();
  NewCond = getSetCCInverse(NewCond, /*isInteger=*/true);
  if (!checkConstants())
    return SDValue();
  // Great, e.g. got  icmp uge i16 (add i16 %x, -128), -256
}

// They are power-of-two, so which bit is set?
const unsigned KeptBits = I1.logBase2();
const unsigned KeptBitsMinusOne = I01.logBase2();

// Magic!
if (KeptBits != (KeptBitsMinusOne + 1))
  return SDValue();
assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable")((KeptBits > 0 && KeptBits < XVT.getSizeInBits(
) && "unreachable") ? static_cast<void> (0) : __assert_fail
 ("KeptBits > 0 && KeptBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2913, __PRETTY_FUNCTION__));

// We don't want to do this in every single case.
SelectionDAG &DAG = DCI.DAG;
if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
        XVT, KeptBits))
  return SDValue();

const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable")((MaskedBits > 0 && MaskedBits < XVT.getSizeInBits
() && "unreachable") ? static_cast<void> (0) : __assert_fail
 ("MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && \"unreachable\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2922, __PRETTY_FUNCTION__));

// Unfold into:  ((%x << C) a>> C) cond %x
// Where 'cond' will be either 'eq' or 'ne'.
SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);

return T2;
2932}

2934// (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
2935SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
  EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
  DAGCombinerInfo &DCI, const SDLoc &DL) const {
assert(isConstOrConstSplat(N1C) &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2940, __PRETTY_FUNCTION__))
       isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2940, __PRETTY_FUNCTION__))
       "Should be a comparison with 0.")((isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C
)->getAPIntValue().isNullValue() && "Should be a comparison with 0."
) ? static_cast<void> (0) : __assert_fail ("isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() && \"Should be a comparison with 0.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2940, __PRETTY_FUNCTION__));
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2942, __PRETTY_FUNCTION__))
       "Valid only for [in]equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Valid only for [in]equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Valid only for [in]equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 2942, __PRETTY_FUNCTION__));

unsigned NewShiftOpcode;
SDValue X, C, Y;

SelectionDAG &DAG = DCI.DAG;
const TargetLowering &TLI = DAG.getTargetLoweringInfo();

// Look for '(C l>>/<< Y)'.
auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
  // The shift should be one-use.
  if (!V.hasOneUse())
    return false;
  unsigned OldShiftOpcode = V.getOpcode();
  switch (OldShiftOpcode) {
  case ISD::SHL:
    NewShiftOpcode = ISD::SRL;
    break;
  case ISD::SRL:
    NewShiftOpcode = ISD::SHL;
    break;
  default:
    return false; // must be a logical shift.
  }
  // We should be shifting a constant.
  // FIXME: best to use isConstantOrConstantVector().
  C = V.getOperand(0);
  ConstantSDNode *CC =
      isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
  if (!CC)
    return false;
  Y = V.getOperand(1);

  ConstantSDNode *XC =
      isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
  return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
      X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
};

// LHS of comparison should be an one-use 'and'.
if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
  return SDValue();

X = N0.getOperand(0);
SDValue Mask = N0.getOperand(1);

// 'and' is commutative!
if (!Match(Mask)) {
  std::swap(X, Mask);
  if (!Match(Mask))
    return SDValue();
}

EVT VT = X.getValueType();

// Produce:
// ((X 'OppositeShiftOpcode' Y) & C) Cond 0
SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
return T2;
3003}

3005/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3006/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3007/// handle the commuted versions of these patterns.
3008SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
                                         ISD::CondCode Cond, const SDLoc &DL,
                                         DAGCombinerInfo &DCI) const {
unsigned BOpcode = N0.getOpcode();
assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
 (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3013, __PRETTY_FUNCTION__))
       "Unexpected binop")(((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD
::XOR) && "Unexpected binop") ? static_cast<void>
 (0) : __assert_fail ("(BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && \"Unexpected binop\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3013, __PRETTY_FUNCTION__));
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Unexpected condcode\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3014, __PRETTY_FUNCTION__));

// (X + Y) == X --> Y == 0
// (X - Y) == X --> Y == 0
// (X ^ Y) == X --> Y == 0
SelectionDAG &DAG = DCI.DAG;
EVT OpVT = N0.getValueType();
SDValue X = N0.getOperand(0);
SDValue Y = N0.getOperand(1);
if (X == N1)
  return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);

if (Y != N1)
  return SDValue();

// (X + Y) == Y --> X == 0
// (X ^ Y) == Y --> X == 0
if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
  return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);

// The shift would not be valid if the operands are boolean (i1).
if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
  return SDValue();

// (X - Y) == Y --> X == Y << 1
EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
                               !DCI.isBeforeLegalize());
SDValue One = DAG.getConstant(1, DL, ShiftVT);
SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
if (!DCI.isCalledByLegalizer())
  DCI.AddToWorklist(YShl1.getNode());
return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3046}

3048/// Try to simplify a setcc built with the specified operands and cc. If it is
3049/// unable to simplify it, return a null SDValue.
3050SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
                                    ISD::CondCode Cond, bool foldBooleans,
                                    DAGCombinerInfo &DCI,
                                    const SDLoc &dl) const {
SelectionDAG &DAG = DCI.DAG;
const DataLayout &Layout = DAG.getDataLayout();
EVT OpVT = N0.getValueType();

// Constant fold or commute setcc.
if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
  return Fold;

// Ensure that the constant occurs on the RHS and fold constant comparisons.
// TODO: Handle non-splat vector constants. All undef causes trouble.
ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
if (isConstOrConstSplat(N0) &&
    (DCI.isBeforeLegalizeOps() ||
     isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

// If we have a subtract with the same 2 non-constant operands as this setcc
// -- but in reverse order -- then try to commute the operands of this setcc
// to match. A matching pair of setcc (cmp) and sub may be combined into 1
// instruction on some targets.
if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
    (DCI.isBeforeLegalizeOps() ||
     isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
    DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N1, N0 } ) &&
    !DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N0, N1 } ))
  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);

if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
  const APInt &C1 = N1C->getAPIntValue();

  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
  // equality comparison, then we're just comparing whether X itself is
  // zero.
  if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
      N0.getOperand(0).getOpcode() == ISD::CTLZ &&
      N0.getOperand(1).getOpcode() == ISD::Constant) {
    const APInt &ShAmt = N0.getConstantOperandAPInt(1);
    if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        ShAmt == Log2_32(N0.getValueSizeInBits())) {
      if ((C1 == 0) == (Cond == ISD::SETEQ)) {
        // (srl (ctlz x), 5) == 0  -> X != 0
        // (srl (ctlz x), 5) != 1  -> X != 0
        Cond = ISD::SETNE;
      } else {
        // (srl (ctlz x), 5) != 0  -> X == 0
        // (srl (ctlz x), 5) == 1  -> X == 0
        Cond = ISD::SETEQ;
      }
      SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
      return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0),
                          Zero, Cond);
    }
  }

  SDValue CTPOP = N0;
  // Look through truncs that don't change the value of a ctpop.
  if (N0.hasOneUse() && N0.getOpcode() == ISD::TRUNCATE)
    CTPOP = N0.getOperand(0);

  if (CTPOP.hasOneUse() && CTPOP.getOpcode() == ISD::CTPOP &&
      (N0 == CTPOP ||
       N0.getValueSizeInBits() > Log2_32_Ceil(CTPOP.getValueSizeInBits()))) {
    EVT CTVT = CTPOP.getValueType();
    SDValue CTOp = CTPOP.getOperand(0);

    // (ctpop x) u< 2 -> (x & x-1) == 0
    // (ctpop x) u> 1 -> (x & x-1) != 0
    if ((Cond == ISD::SETULT && C1 == 2) || (Cond == ISD::SETUGT && C1 == 1)){
      SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
      SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
      SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
      ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
      return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC);
    }

    // If ctpop is not supported, expand a power-of-2 comparison based on it.
    if (C1 == 1 && !isOperationLegalOrCustom(ISD::CTPOP, CTVT) &&
        (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
      // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
      // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
      SDValue Zero = DAG.getConstant(0, dl, CTVT);
      SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
      ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, true);
      SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
      SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
      SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
      SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
      unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
      return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
    }
  }

  // (zext x) == C --> x == (trunc C)
  // (sext x) == C --> x == (trunc C)
  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
      DCI.isBeforeLegalize() && N0->hasOneUse()) {
    unsigned MinBits = N0.getValueSizeInBits();
    SDValue PreExt;
    bool Signed = false;
    if (N0->getOpcode() == ISD::ZERO_EXTEND) {
      // ZExt
      MinBits = N0->getOperand(0).getValueSizeInBits();
      PreExt = N0->getOperand(0);
    } else if (N0->getOpcode() == ISD::AND) {
      // DAGCombine turns costly ZExts into ANDs
      if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
        if ((C->getAPIntValue()+1).isPowerOf2()) {
          MinBits = C->getAPIntValue().countTrailingOnes();
          PreExt = N0->getOperand(0);
        }
    } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
      // SExt
      MinBits = N0->getOperand(0).getValueSizeInBits();
      PreExt = N0->getOperand(0);
      Signed = true;
    } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
      // ZEXTLOAD / SEXTLOAD
      if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
        MinBits = LN0->getMemoryVT().getSizeInBits();
        PreExt = N0;
      } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
        Signed = true;
        MinBits = LN0->getMemoryVT().getSizeInBits();
        PreExt = N0;
      }
    }

    // Figure out how many bits we need to preserve this constant.
    unsigned ReqdBits = Signed ?
      C1.getBitWidth() - C1.getNumSignBits() + 1 :
      C1.getActiveBits();

    // Make sure we're not losing bits from the constant.
    if (MinBits > 0 &&
        MinBits < C1.getBitWidth() &&
        MinBits >= ReqdBits) {
      EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
      if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
        // Will get folded away.
        SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
        if (MinBits == 1 && C1 == 1)
          // Invert the condition.
          return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
                              Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
        SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
        return DAG.getSetCC(dl, VT, Trunc, C, Cond);
      }

      // If truncating the setcc operands is not desirable, we can still
      // simplify the expression in some cases:
      // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
      // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
      // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
      // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
      // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
      // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
      SDValue TopSetCC = N0->getOperand(0);
      unsigned N0Opc = N0->getOpcode();
      bool SExt = (N0Opc == ISD::SIGN_EXTEND);
      if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
          TopSetCC.getOpcode() == ISD::SETCC &&
          (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
          (isConstFalseVal(N1C) ||
           isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {

        bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
                       (!N1C->isNullValue() && Cond == ISD::SETNE);

        if (!Inverse)
          return TopSetCC;

        ISD::CondCode InvCond = ISD::getSetCCInverse(
            cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
            TopSetCC.getOperand(0).getValueType().isInteger());
        return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
                                    TopSetCC.getOperand(1),
                                    InvCond);
      }
    }
  }

  // If the LHS is '(and load, const)', the RHS is 0, the test is for
  // equality or unsigned, and all 1 bits of the const are in the same
  // partial word, see if we can shorten the load.
  if (DCI.isBeforeLegalize() &&
      !ISD::isSignedIntSetCC(Cond) &&
      N0.getOpcode() == ISD::AND && C1 == 0 &&
      N0.getNode()->hasOneUse() &&
      isa<LoadSDNode>(N0.getOperand(0)) &&
      N0.getOperand(0).getNode()->hasOneUse() &&
      isa<ConstantSDNode>(N0.getOperand(1))) {
    LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
    APInt bestMask;
    unsigned bestWidth = 0, bestOffset = 0;
    if (Lod->isSimple() && Lod->isUnindexed()) {
      unsigned origWidth = N0.getValueSizeInBits();
      unsigned maskWidth = origWidth;
      // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
      // 8 bits, but have to be careful...
      if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
        origWidth = Lod->getMemoryVT().getSizeInBits();
      const APInt &Mask = N0.getConstantOperandAPInt(1);
      for (unsigned width = origWidth / 2; width>=8; width /= 2) {
        APInt newMask = APInt::getLowBitsSet(maskWidth, width);
        for (unsigned offset=0; offset<origWidth/width; offset++) {
          if (Mask.isSubsetOf(newMask)) {
            if (Layout.isLittleEndian())
              bestOffset = (uint64_t)offset * (width/8);
            else
              bestOffset = (origWidth/width - offset - 1) * (width/8);
            bestMask = Mask.lshr(offset * (width/8) * 8);
            bestWidth = width;
            break;
          }
          newMask <<= width;
        }
      }
    }
    if (bestWidth) {
      EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
      if (newVT.isRound() &&
          shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
        EVT PtrType = Lod->getOperand(1).getValueType();
        SDValue Ptr = Lod->getBasePtr();
        if (bestOffset != 0)
          Ptr = DAG.getNode(ISD::ADD, dl, PtrType, Lod->getBasePtr(),
                            DAG.getConstant(bestOffset, dl, PtrType));
        unsigned NewAlign = MinAlign(Lod->getAlignment(), bestOffset);
        SDValue NewLoad = DAG.getLoad(
            newVT, dl, Lod->getChain(), Ptr,
            Lod->getPointerInfo().getWithOffset(bestOffset), NewAlign);
        return DAG.getSetCC(dl, VT,
                            DAG.getNode(ISD::AND, dl, newVT, NewLoad,
                                    DAG.getConstant(bestMask.trunc(bestWidth),
                                                    dl, newVT)),
                            DAG.getConstant(0LL, dl, newVT), Cond);
      }
    }
  }

  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
    unsigned InSize = N0.getOperand(0).getValueSizeInBits();

    // If the comparison constant has bits in the upper part, the
    // zero-extended value could never match.
    if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
                                            C1.getBitWidth() - InSize))) {
      switch (Cond) {
      case ISD::SETUGT:
      case ISD::SETUGE:
      case ISD::SETEQ:
        return DAG.getConstant(0, dl, VT);
      case ISD::SETULT:
      case ISD::SETULE:
      case ISD::SETNE:
        return DAG.getConstant(1, dl, VT);
      case ISD::SETGT:
      case ISD::SETGE:
        // True if the sign bit of C1 is set.
        return DAG.getConstant(C1.isNegative(), dl, VT);
      case ISD::SETLT:
      case ISD::SETLE:
        // True if the sign bit of C1 isn't set.
        return DAG.getConstant(C1.isNonNegative(), dl, VT);
      default:
        break;
      }
    }

    // Otherwise, we can perform the comparison with the low bits.
    switch (Cond) {
    case ISD::SETEQ:
    case ISD::SETNE:
    case ISD::SETUGT:
    case ISD::SETUGE:
    case ISD::SETULT:
    case ISD::SETULE: {
      EVT newVT = N0.getOperand(0).getValueType();
      if (DCI.isBeforeLegalizeOps() ||
          (isOperationLegal(ISD::SETCC, newVT) &&
           isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
        EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
        SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);

        SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
                                        NewConst, Cond);
        return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
      }
      break;
    }
    default:
      break; // todo, be more careful with signed comparisons
    }
  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
             (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
    EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
    unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
    EVT ExtDstTy = N0.getValueType();
    unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();

    // If the constant doesn't fit into the number of bits for the source of
    // the sign extension, it is impossible for both sides to be equal.
    if (C1.getMinSignedBits() > ExtSrcTyBits)
      return DAG.getConstant(Cond == ISD::SETNE, dl, VT);

    SDValue ZextOp;
    EVT Op0Ty = N0.getOperand(0).getValueType();
    if (Op0Ty == ExtSrcTy) {
      ZextOp = N0.getOperand(0);
    } else {
      APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
      ZextOp = DAG.getNode(ISD::AND, dl, Op0Ty, N0.getOperand(0),
                           DAG.getConstant(Imm, dl, Op0Ty));
    }
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(ZextOp.getNode());
    // Otherwise, make this a use of a zext.
    return DAG.getSetCC(dl, VT, ZextOp,
                        DAG.getConstant(C1 & APInt::getLowBitsSet(
                                                            ExtDstTyBits,
                                                            ExtSrcTyBits),
                                        dl, ExtDstTy),
                        Cond);
  } else if ((N1C->isNullValue() || N1C->isOne()) &&
              (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
    // SETCC (SETCC), [0|1], [EQ|NE]  -> SETCC
    if (N0.getOpcode() == ISD::SETCC &&
        isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
        (N0.getValueType() == MVT::i1 ||
         getBooleanContents(N0.getOperand(0).getValueType()) ==
                     ZeroOrOneBooleanContent)) {
      bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
      if (TrueWhenTrue)
        return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
      // Invert the condition.
      ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
      CC = ISD::getSetCCInverse(CC,
                                N0.getOperand(0).getValueType().isInteger());
      if (DCI.isBeforeLegalizeOps() ||
          isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
        return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
    }

    if ((N0.getOpcode() == ISD::XOR ||
         (N0.getOpcode() == ISD::AND &&
          N0.getOperand(0).getOpcode() == ISD::XOR &&
          N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
        isa<ConstantSDNode>(N0.getOperand(1)) &&
        cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
      // If this is (X^1) == 0/1, swap the RHS and eliminate the xor.  We
      // can only do this if the top bits are known zero.
      unsigned BitWidth = N0.getValueSizeInBits();
      if (DAG.MaskedValueIsZero(N0,
                                APInt::getHighBitsSet(BitWidth,
                                                      BitWidth-1))) {
        // Okay, get the un-inverted input value.
        SDValue Val;
        if (N0.getOpcode() == ISD::XOR) {
          Val = N0.getOperand(0);
        } else {
          assert(N0.getOpcode() == ISD::AND &&((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
 ("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3416, __PRETTY_FUNCTION__))
                  N0.getOperand(0).getOpcode() == ISD::XOR)((N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode
() == ISD::XOR) ? static_cast<void> (0) : __assert_fail
 ("N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::XOR"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3416, __PRETTY_FUNCTION__));
          // ((X^1)&1)^1 -> X & 1
          Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
                            N0.getOperand(0).getOperand(0),
                            N0.getOperand(1));
        }

        return DAG.getSetCC(dl, VT, Val, N1,
                            Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
      }
    } else if (N1C->isOne()) {
      SDValue Op0 = N0;
      if (Op0.getOpcode() == ISD::TRUNCATE)
        Op0 = Op0.getOperand(0);

      if ((Op0.getOpcode() == ISD::XOR) &&
          Op0.getOperand(0).getOpcode() == ISD::SETCC &&
          Op0.getOperand(1).getOpcode() == ISD::SETCC) {
        SDValue XorLHS = Op0.getOperand(0);
        SDValue XorRHS = Op0.getOperand(1);
        // Ensure that the input setccs return an i1 type or 0/1 value.
        if (Op0.getValueType() == MVT::i1 ||
            (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
                    ZeroOrOneBooleanContent &&
             getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
                      ZeroOrOneBooleanContent)) {
          // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
          Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
          return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
        }
      }
      if (Op0.getOpcode() == ISD::AND &&
          isa<ConstantSDNode>(Op0.getOperand(1)) &&
          cast<ConstantSDNode>(Op0.getOperand(1))->isOne()) {
        // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
        if (Op0.getValueType().bitsGT(VT))
          Op0 = DAG.getNode(ISD::AND, dl, VT,
                        DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
                        DAG.getConstant(1, dl, VT));
        else if (Op0.getValueType().bitsLT(VT))
          Op0 = DAG.getNode(ISD::AND, dl, VT,
                      DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
                      DAG.getConstant(1, dl, VT));

        return DAG.getSetCC(dl, VT, Op0,
                            DAG.getConstant(0, dl, Op0.getValueType()),
                            Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
      }
      if (Op0.getOpcode() == ISD::AssertZext &&
          cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
        return DAG.getSetCC(dl, VT, Op0,
                            DAG.getConstant(0, dl, Op0.getValueType()),
                            Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
    }
  }

  // Given:
  //   icmp eq/ne (urem %x, %y), 0
  // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
  //   icmp eq/ne %x, 0
  if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
      (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
    KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
    KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
    if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
      return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
  }

  if (SDValue V =
          optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
    return V;
}

// These simplifications apply to splat vectors as well.
// TODO: Handle more splat vector cases.
if (auto *N1C = isConstOrConstSplat(N1)) {
  const APInt &C1 = N1C->getAPIntValue();

  APInt MinVal, MaxVal;
  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
  if (ISD::isSignedIntSetCC(Cond)) {
    MinVal = APInt::getSignedMinValue(OperandBitSize);
    MaxVal = APInt::getSignedMaxValue(OperandBitSize);
  } else {
    MinVal = APInt::getMinValue(OperandBitSize);
    MaxVal = APInt::getMaxValue(OperandBitSize);
  }

  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
    // X >= MIN --> true
    if (C1 == MinVal)
      return DAG.getBoolConstant(true, dl, VT, OpVT);

    if (!VT.isVector()) { // TODO: Support this for vectors.
      // X >= C0 --> X > (C0 - 1)
      APInt C = C1 - 1;
      ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
      if ((DCI.isBeforeLegalizeOps() ||
           isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
          (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
                                isLegalICmpImmediate(C.getSExtValue())))) {
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(C, dl, N1.getValueType()),
                            NewCC);
      }
    }
  }

  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
    // X <= MAX --> true
    if (C1 == MaxVal)
      return DAG.getBoolConstant(true, dl, VT, OpVT);

    // X <= C0 --> X < (C0 + 1)
    if (!VT.isVector()) { // TODO: Support this for vectors.
      APInt C = C1 + 1;
      ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
      if ((DCI.isBeforeLegalizeOps() ||
           isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
          (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
                                isLegalICmpImmediate(C.getSExtValue())))) {
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(C, dl, N1.getValueType()),
                            NewCC);
      }
    }
  }

  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
    if (C1 == MinVal)
      return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false

    // TODO: Support this for vectors after legalize ops.
    if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
      // Canonicalize setlt X, Max --> setne X, Max
      if (C1 == MaxVal)
        return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

      // If we have setult X, 1, turn it into seteq X, 0
      if (C1 == MinVal+1)
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(MinVal, dl, N0.getValueType()),
                            ISD::SETEQ);
    }
  }

  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
    if (C1 == MaxVal)
      return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false

    // TODO: Support this for vectors after legalize ops.
    if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
      // Canonicalize setgt X, Min --> setne X, Min
      if (C1 == MinVal)
        return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);

      // If we have setugt X, Max-1, turn it into seteq X, Max
      if (C1 == MaxVal-1)
        return DAG.getSetCC(dl, VT, N0,
                            DAG.getConstant(MaxVal, dl, N0.getValueType()),
                            ISD::SETEQ);
    }
  }

  if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
    // (X & (C l>>/<< Y)) ==/!= 0  -->  ((X <</l>> Y) & C) ==/!= 0
    if (C1.isNullValue())
      if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
              VT, N0, N1, Cond, DCI, dl))
        return CC;
  }

  // If we have "setcc X, C0", check to see if we can shrink the immediate
  // by changing cc.
  // TODO: Support this for vectors after legalize ops.
  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
    // SETUGT X, SINTMAX  -> SETLT X, 0
    if (Cond == ISD::SETUGT &&
        C1 == APInt::getSignedMaxValue(OperandBitSize))
      return DAG.getSetCC(dl, VT, N0,
                          DAG.getConstant(0, dl, N1.getValueType()),
                          ISD::SETLT);

    // SETULT X, SINTMIN  -> SETGT X, -1
    if (Cond == ISD::SETULT &&
        C1 == APInt::getSignedMinValue(OperandBitSize)) {
      SDValue ConstMinusOne =
          DAG.getConstant(APInt::getAllOnesValue(OperandBitSize), dl,
                          N1.getValueType());
      return DAG.getSetCC(dl, VT, N0, ConstMinusOne, ISD::SETGT);
    }
  }
}

// Back to non-vector simplifications.
// TODO: Can we do these for vector splats?
if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  const APInt &C1 = N1C->getAPIntValue();
  EVT ShValTy = N0.getValueType();

  // Fold bit comparisons when we can.
  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
      (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
      N0.getOpcode() == ISD::AND) {
    if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
      EVT ShiftTy =
          getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
      if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0  -->  (X & 8) >> 3
        // Perform the xform if the AND RHS is a single bit.
        unsigned ShCt = AndRHS->getAPIntValue().logBase2();
        if (AndRHS->getAPIntValue().isPowerOf2() &&
            ShCt <= TLI.getShiftAmountThreshold(ShValTy)) {
          return DAG.getNode(ISD::TRUNCATE, dl, VT,
                             DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                                         DAG.getConstant(ShCt, dl, ShiftTy)));
        }
      } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
        // (X & 8) == 8  -->  (X & 8) >> 3
        // Perform the xform if C1 is a single bit.
        unsigned ShCt = C1.logBase2();
        if (C1.isPowerOf2() &&
            ShCt <= TLI.getShiftAmountThreshold(ShValTy)) {
          return DAG.getNode(ISD::TRUNCATE, dl, VT,
                             DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                                         DAG.getConstant(ShCt, dl, ShiftTy)));
        }
      }
    }
  }

  if (C1.getMinSignedBits() <= 64 &&
      !isLegalICmpImmediate(C1.getSExtValue())) {
    EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
    // (X & -256) == 256 -> (X >> 8) == 1
    if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
        N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
      if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
        const APInt &AndRHSC = AndRHS->getAPIntValue();
        if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
          unsigned ShiftBits = AndRHSC.countTrailingZeros();
          if (ShiftBits <= TLI.getShiftAmountThreshold(ShValTy)) {
            SDValue Shift =
              DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
                          DAG.getConstant(ShiftBits, dl, ShiftTy));
            SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
            return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
          }
        }
      }
    } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
               Cond == ISD::SETULE || Cond == ISD::SETUGT) {
      bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
      // X <  0x100000000 -> (X >> 32) <  1
      // X >= 0x100000000 -> (X >> 32) >= 1
      // X <= 0x0ffffffff -> (X >> 32) <  1
      // X >  0x0ffffffff -> (X >> 32) >= 1
      unsigned ShiftBits;
      APInt NewC = C1;
      ISD::CondCode NewCond = Cond;
      if (AdjOne) {
        ShiftBits = C1.countTrailingOnes();
        NewC = NewC + 1;
        NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
      } else {
        ShiftBits = C1.countTrailingZeros();
      }
      NewC.lshrInPlace(ShiftBits);
      if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
          isLegalICmpImmediate(NewC.getSExtValue()) &&
          ShiftBits <= TLI.getShiftAmountThreshold(ShValTy)) {
        SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                                    DAG.getConstant(ShiftBits, dl, ShiftTy));
        SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
        return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
      }
    }
  }
}

if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
  auto *CFP = cast<ConstantFPSDNode>(N1);
  assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value")((!CFP->getValueAPF().isNaN() && "Unexpected NaN value"
) ? static_cast<void> (0) : __assert_fail ("!CFP->getValueAPF().isNaN() && \"Unexpected NaN value\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3699, __PRETTY_FUNCTION__));

  // Otherwise, we know the RHS is not a NaN.  Simplify the node to drop the
  // constant if knowing that the operand is non-nan is enough.  We prefer to
  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
  // materialize 0.0.
  if (Cond == ISD::SETO || Cond == ISD::SETUO)
    return DAG.getSetCC(dl, VT, N0, N0, Cond);

  // setcc (fneg x), C -> setcc swap(pred) x, -C
  if (N0.getOpcode() == ISD::FNEG) {
    ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
    if (DCI.isBeforeLegalizeOps() ||
        isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
      SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
      return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
    }
  }

  // If the condition is not legal, see if we can find an equivalent one
  // which is legal.
  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
    // If the comparison was an awkward floating-point == or != and one of
    // the comparison operands is infinity or negative infinity, convert the
    // condition to a less-awkward <= or >=.
    if (CFP->getValueAPF().isInfinity()) {
      if (CFP->getValueAPF().isNegative()) {
        if (Cond == ISD::SETOEQ &&
            isCondCodeLegal(ISD::SETOLE, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLE);
        if (Cond == ISD::SETUEQ &&
            isCondCodeLegal(ISD::SETOLE, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULE);
        if (Cond == ISD::SETUNE &&
            isCondCodeLegal(ISD::SETUGT, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGT);
        if (Cond == ISD::SETONE &&
            isCondCodeLegal(ISD::SETUGT, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGT);
      } else {
        if (Cond == ISD::SETOEQ &&
            isCondCodeLegal(ISD::SETOGE, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOGE);
        if (Cond == ISD::SETUEQ &&
            isCondCodeLegal(ISD::SETOGE, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETUGE);
        if (Cond == ISD::SETUNE &&
            isCondCodeLegal(ISD::SETULT, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETULT);
        if (Cond == ISD::SETONE &&
            isCondCodeLegal(ISD::SETULT, N0.getSimpleValueType()))
          return DAG.getSetCC(dl, VT, N0, N1, ISD::SETOLT);
      }
    }
  }
}

if (N0 == N1) {
  // The sext(setcc()) => setcc() optimization relies on the appropriate
  // constant being emitted.
  assert(!N0.getValueType().isInteger() &&((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3760, __PRETTY_FUNCTION__))
         "Integer types should be handled by FoldSetCC")((!N0.getValueType().isInteger() && "Integer types should be handled by FoldSetCC"
) ? static_cast<void> (0) : __assert_fail ("!N0.getValueType().isInteger() && \"Integer types should be handled by FoldSetCC\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3760, __PRETTY_FUNCTION__));

  bool EqTrue = ISD::isTrueWhenEqual(Cond);
  unsigned UOF = ISD::getUnorderedFlavor(Cond);
  if (UOF == 2) // FP operators that are undefined on NaNs.
    return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
  if (UOF == unsigned(EqTrue))
    return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
  // Otherwise, we can't fold it.  However, we can simplify it to SETUO/SETO
  // if it is not already.
  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
  if (NewCond != Cond &&
      (DCI.isBeforeLegalizeOps() ||
                          isCondCodeLegal(NewCond, N0.getSimpleValueType())))
    return DAG.getSetCC(dl, VT, N0, N1, NewCond);
}

if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
    N0.getValueType().isInteger()) {
  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
      N0.getOpcode() == ISD::XOR) {
    // Simplify (X+Y) == (X+Z) -->  Y == Z
    if (N0.getOpcode() == N1.getOpcode()) {
      if (N0.getOperand(0) == N1.getOperand(0))
        return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
      if (N0.getOperand(1) == N1.getOperand(1))
        return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
      if (isCommutativeBinOp(N0.getOpcode())) {
        // If X op Y == Y op X, try other combinations.
        if (N0.getOperand(0) == N1.getOperand(1))
          return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
                              Cond);
        if (N0.getOperand(1) == N1.getOperand(0))
          return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
                              Cond);
      }
    }

    // If RHS is a legal immediate value for a compare instruction, we need
    // to be careful about increasing register pressure needlessly.
    bool LegalRHSImm = false;

    if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
      if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
        // Turn (X+C1) == C2 --> X == C2-C1
        if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
          return DAG.getSetCC(dl, VT, N0.getOperand(0),
                              DAG.getConstant(RHSC->getAPIntValue()-
                                              LHSR->getAPIntValue(),
                              dl, N0.getValueType()), Cond);
        }

        // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
        if (N0.getOpcode() == ISD::XOR)
          // If we know that all of the inverted bits are zero, don't bother
          // performing the inversion.
          if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
            return
              DAG.getSetCC(dl, VT, N0.getOperand(0),
                           DAG.getConstant(LHSR->getAPIntValue() ^
                                             RHSC->getAPIntValue(),
                                           dl, N0.getValueType()),
                           Cond);
      }

      // Turn (C1-X) == C2 --> X == C1-C2
      if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
        if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
          return
            DAG.getSetCC(dl, VT, N0.getOperand(1),
                         DAG.getConstant(SUBC->getAPIntValue() -
                                           RHSC->getAPIntValue(),
                                         dl, N0.getValueType()),
                         Cond);
        }
      }

      // Could RHSC fold directly into a compare?
      if (RHSC->getValueType(0).getSizeInBits() <= 64)
        LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
    }

    // (X+Y) == X --> Y == 0 and similar folds.
    // Don't do this if X is an immediate that can fold into a cmp
    // instruction and X+Y has other uses. It could be an induction variable
    // chain, and the transform would increase register pressure.
    if (!LegalRHSImm || N0.hasOneUse())
      if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
        return V;
  }

  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
      N1.getOpcode() == ISD::XOR)
    if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
      return V;

  if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
    return V;
}

// Fold remainder of division by a constant.
if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
    N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();

  // When division is cheap or optimizing for minimum size,
  // fall through to DIVREM creation by skipping this fold.
  if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
    if (N0.getOpcode() == ISD::UREM) {
      if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
        return Folded;
    } else if (N0.getOpcode() == ISD::SREM) {
      if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
        return Folded;
    }
  }
}

// Fold away ALL boolean setcc's.
if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
  SDValue Temp;
  switch (Cond) {
  default: llvm_unreachable("Unknown integer setcc!")::llvm::llvm_unreachable_internal("Unknown integer setcc!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 3882);
  case ISD::SETEQ:  // X == Y  -> ~(X^Y)
    Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
    N0 = DAG.getNOT(dl, Temp, OpVT);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETNE:  // X != Y   -->  (X^Y)
    N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
    break;
  case ISD::SETGT:  // X >s Y   -->  X == 0 & Y == 1  -->  ~X & Y
  case ISD::SETULT: // X <u Y   -->  X == 0 & Y == 1  -->  ~X & Y
    Temp = DAG.getNOT(dl, N0, OpVT);
    N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETLT:  // X <s Y   --> X == 1 & Y == 0  -->  ~Y & X
  case ISD::SETUGT: // X >u Y   --> X == 1 & Y == 0  -->  ~Y & X
    Temp = DAG.getNOT(dl, N1, OpVT);
    N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETULE: // X <=u Y  --> X == 0 | Y == 1  -->  ~X | Y
  case ISD::SETGE:  // X >=s Y  --> X == 0 | Y == 1  -->  ~X | Y
    Temp = DAG.getNOT(dl, N0, OpVT);
    N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(Temp.getNode());
    break;
  case ISD::SETUGE: // X >=u Y  --> X == 1 | Y == 0  -->  ~Y | X
  case ISD::SETLE:  // X <=s Y  --> X == 1 | Y == 0  -->  ~Y | X
    Temp = DAG.getNOT(dl, N1, OpVT);
    N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
    break;
  }
  if (VT.getScalarType() != MVT::i1) {
    if (!DCI.isCalledByLegalizer())
      DCI.AddToWorklist(N0.getNode());
    // FIXME: If running after legalize, we probably can't do this.
    ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
    N0 = DAG.getNode(ExtendCode, dl, VT, N0);
  }
  return N0;
}

// Could not fold it.
return SDValue();
3931}

3933/// Returns true (and the GlobalValue and the offset) if the node is a
3934/// GlobalAddress + offset.
3935bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
                                  int64_t &Offset) const {

SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();

if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
  GA = GASD->getGlobal();
  Offset += GASD->getOffset();
  return true;
}

if (N->getOpcode() == ISD::ADD) {
  SDValue N1 = N->getOperand(0);
  SDValue N2 = N->getOperand(1);
  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
    if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
      Offset += V->getSExtValue();
      return true;
    }
  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
    if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
      Offset += V->getSExtValue();
      return true;
    }
  }
}

return false;
3963}

3965SDValue TargetLowering::PerformDAGCombine(SDNode *N,
                                        DAGCombinerInfo &DCI) const {
// Default implementation: no optimization.
return SDValue();
3969}

3971//===----------------------------------------------------------------------===//
3972//  Inline Assembler Implementation Methods
3973//===----------------------------------------------------------------------===//

3975TargetLowering::ConstraintType
3976TargetLowering::getConstraintType(StringRef Constraint) const {
unsigned S = Constraint.size();

if (S == 1) {
  switch (Constraint[0]) {
  default: break;
  case 'r':
    return C_RegisterClass;
  case 'm': // memory
  case 'o': // offsetable
  case 'V': // not offsetable
    return C_Memory;
  case 'n': // Simple Integer
  case 'E': // Floating Point Constant
  case 'F': // Floating Point Constant
    return C_Immediate;
  case 'i': // Simple Integer or Relocatable Constant
  case 's': // Relocatable Constant
  case 'p': // Address.
  case 'X': // Allow ANY value.
  case 'I': // Target registers.
  case 'J':
  case 'K':
  case 'L':
  case 'M':
  case 'N':
  case 'O':
  case 'P':
  case '<':
  case '>':
    return C_Other;
  }
}

if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
  if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
    return C_Memory;
  return C_Register;
}
return C_Unknown;
4016}

4018/// Try to replace an X constraint, which matches anything, with another that
4019/// has more specific requirements based on the type of the corresponding
4020/// operand.
4021const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
if (ConstraintVT.isInteger())
  return "r";
if (ConstraintVT.isFloatingPoint())
  return "f"; // works for many targets
return nullptr;
4027}

4029SDValue TargetLowering::LowerAsmOutputForConstraint(
  SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
  SelectionDAG &DAG) const {
return SDValue();
4033}

4035/// Lower the specified operand into the Ops vector.
4036/// If it is invalid, don't add anything to Ops.
4037void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
                                                std::string &Constraint,
                                                std::vector<SDValue> &Ops,
                                                SelectionDAG &DAG) const {

if (Constraint.length() > 1) return;

char ConstraintLetter = Constraint[0];
switch (ConstraintLetter) {
default: break;
case 'X':     // Allows any operand; labels (basic block) use this.
  if (Op.getOpcode() == ISD::BasicBlock ||
      Op.getOpcode() == ISD::TargetBlockAddress) {
    Ops.push_back(Op);
    return;
  }
  LLVM_FALLTHROUGH[[gnu::fallthrough]];
case 'i':    // Simple Integer or Relocatable Constant
case 'n':    // Simple Integer
case 's': {  // Relocatable Constant

  GlobalAddressSDNode *GA;
  ConstantSDNode *C;
  BlockAddressSDNode *BA;
  uint64_t Offset = 0;

  // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
  // etc., since getelementpointer is variadic. We can't use
  // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
  // while in this case the GA may be furthest from the root node which is
  // likely an ISD::ADD.
  while (1) {
    if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
      Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
                                               GA->getValueType(0),
                                               Offset + GA->getOffset()));
      return;
    } else if ((C = dyn_cast<ConstantSDNode>(Op)) &&
               ConstraintLetter != 's') {
      // gcc prints these as sign extended.  Sign extend value to 64 bits
      // now; without this it would get ZExt'd later in
      // ScheduleDAGSDNodes::EmitNode, which is very generic.
      bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
      BooleanContent BCont = getBooleanContents(MVT::i64);
      ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
                                    : ISD::SIGN_EXTEND;
      int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue()
                                                  : C->getSExtValue();
      Ops.push_back(DAG.getTargetConstant(Offset + ExtVal,
                                          SDLoc(C), MVT::i64));
      return;
    } else if ((BA = dyn_cast<BlockAddressSDNode>(Op)) &&
               ConstraintLetter != 'n') {
      Ops.push_back(DAG.getTargetBlockAddress(
          BA->getBlockAddress(), BA->getValueType(0),
          Offset + BA->getOffset(), BA->getTargetFlags()));
      return;
    } else {
      const unsigned OpCode = Op.getOpcode();
      if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
        if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
          Op = Op.getOperand(1);
        // Subtraction is not commutative.
        else if (OpCode == ISD::ADD &&
                 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
          Op = Op.getOperand(0);
        else
          return;
        Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
        continue;
      }
    }
    return;
  }
  break;
}
}
4114}

4116std::pair<unsigned, const TargetRegisterClass *>
4117TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
                                           StringRef Constraint,
                                           MVT VT) const {
if (Constraint.empty() || Constraint[0] != '{')
  return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?")((*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?"
) ? static_cast<void> (0) : __assert_fail ("*(Constraint.end() - 1) == '}' && \"Not a brace enclosed constraint?\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4122, __PRETTY_FUNCTION__));

// Remove the braces from around the name.
StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);

std::pair<unsigned, const TargetRegisterClass *> R =
    std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));

// Figure out which register class contains this reg.
for (const TargetRegisterClass *RC : RI->regclasses()) {
  // If none of the value types for this register class are valid, we
  // can't use it.  For example, 64-bit reg classes on 32-bit targets.
  if (!isLegalRC(*RI, *RC))
    continue;

  for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end();
       I != E; ++I) {
    if (RegName.equals_lower(RI->getRegAsmName(*I))) {
      std::pair<unsigned, const TargetRegisterClass *> S =
          std::make_pair(*I, RC);

      // If this register class has the requested value type, return it,
      // otherwise keep searching and return the first class found
      // if no other is found which explicitly has the requested type.
      if (RI->isTypeLegalForClass(*RC, VT))
        return S;
      if (!R.second)
        R = S;
    }
  }
}

return R;
4155}

4157//===----------------------------------------------------------------------===//
4158// Constraint Selection.

4160/// Return true of this is an input operand that is a matching constraint like
4161/// "4".
4162bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
 static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4163, __PRETTY_FUNCTION__));
return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4165}

4167/// If this is an input matching constraint, this method returns the output
4168/// operand it matches.
4169unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
assert(!ConstraintCode.empty() && "No known constraint!")((!ConstraintCode.empty() && "No known constraint!") ?
 static_cast<void> (0) : __assert_fail ("!ConstraintCode.empty() && \"No known constraint!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4170, __PRETTY_FUNCTION__));
return atoi(ConstraintCode.c_str());
4172}

4174/// Split up the constraint string from the inline assembly value into the
4175/// specific constraints and their prefixes, and also tie in the associated
4176/// operand values.
4177/// If this returns an empty vector, and if the constraint string itself
4178/// isn't empty, there was an error parsing.
4179TargetLowering::AsmOperandInfoVector
4180TargetLowering::ParseConstraints(const DataLayout &DL,
                               const TargetRegisterInfo *TRI,
                               ImmutableCallSite CS) const {
/// Information about all of the constraints.
AsmOperandInfoVector ConstraintOperands;
const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
unsigned maCount = 0; // Largest number of multiple alternative constraints.

// Do a prepass over the constraints, canonicalizing them, and building up the
// ConstraintOperands list.
unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
unsigned ResNo = 0; // ResNo - The result number of the next output.

for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
  ConstraintOperands.emplace_back(std::move(CI));
  AsmOperandInfo &OpInfo = ConstraintOperands.back();

  // Update multiple alternative constraint count.
  if (OpInfo.multipleAlternatives.size() > maCount)
    maCount = OpInfo.multipleAlternatives.size();

  OpInfo.ConstraintVT = MVT::Other;

  // Compute the value type for each operand.
  switch (OpInfo.Type) {
  case InlineAsm::isOutput:
    // Indirect outputs just consume an argument.
    if (OpInfo.isIndirect) {
      OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
      break;
    }

    // The return value of the call is this value.  As such, there is no
    // corresponding argument.
    assert(!CS.getType()->isVoidTy() &&((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
 static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4215, __PRETTY_FUNCTION__))
           "Bad inline asm!")((!CS.getType()->isVoidTy() && "Bad inline asm!") ?
 static_cast<void> (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4215, __PRETTY_FUNCTION__));
    if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
      OpInfo.ConstraintVT =
          getSimpleValueType(DL, STy->getElementType(ResNo));
    } else {
      assert(ResNo == 0 && "Asm only has one result!")((ResNo == 0 && "Asm only has one result!") ? static_cast
<void> (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4220, __PRETTY_FUNCTION__));
      OpInfo.ConstraintVT = getSimpleValueType(DL, CS.getType());
    }
    ++ResNo;
    break;
  case InlineAsm::isInput:
    OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
    break;
  case InlineAsm::isClobber:
    // Nothing to do.
    break;
  }

  if (OpInfo.CallOperandVal) {
    llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
    if (OpInfo.isIndirect) {
      llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
      if (!PtrTy)
        report_fatal_error("Indirect operand for inline asm not a pointer!");
      OpTy = PtrTy->getElementType();
    }

    // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
    if (StructType *STy = dyn_cast<StructType>(OpTy))
      if (STy->getNumElements() == 1)
        OpTy = STy->getElementType(0);

    // If OpTy is not a single value, it may be a struct/union that we
    // can tile with integers.
    if (!OpTy->isSingleValueType() && OpTy->isSized()) {
      unsigned BitSize = DL.getTypeSizeInBits(OpTy);
      switch (BitSize) {
      default: break;
      case 1:
      case 8:
      case 16:
      case 32:
      case 64:
      case 128:
        OpInfo.ConstraintVT =
            MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
        break;
      }
    } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
      unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
      OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
    } else {
      OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
    }
  }
}

// If we have multiple alternative constraints, select the best alternative.
if (!ConstraintOperands.empty()) {
  if (maCount) {
    unsigned bestMAIndex = 0;
    int bestWeight = -1;
    // weight:  -1 = invalid match, and 0 = so-so match to 5 = good match.
    int weight = -1;
    unsigned maIndex;
    // Compute the sums of the weights for each alternative, keeping track
    // of the best (highest weight) one so far.
    for (maIndex = 0; maIndex < maCount; ++maIndex) {
      int weightSum = 0;
      for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
           cIndex != eIndex; ++cIndex) {
        AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
        if (OpInfo.Type == InlineAsm::isClobber)
          continue;

        // If this is an output operand with a matching input operand,
        // look up the matching input. If their types mismatch, e.g. one
        // is an integer, the other is floating point, or their sizes are
        // different, flag it as an maCantMatch.
        if (OpInfo.hasMatchingInput()) {
          AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
          if (OpInfo.ConstraintVT != Input.ConstraintVT) {
            if ((OpInfo.ConstraintVT.isInteger() !=
                 Input.ConstraintVT.isInteger()) ||
                (OpInfo.ConstraintVT.getSizeInBits() !=
                 Input.ConstraintVT.getSizeInBits())) {
              weightSum = -1; // Can't match.
              break;
            }
          }
        }
        weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
        if (weight == -1) {
          weightSum = -1;
          break;
        }
        weightSum += weight;
      }
      // Update best.
      if (weightSum > bestWeight) {
        bestWeight = weightSum;
        bestMAIndex = maIndex;
      }
    }

    // Now select chosen alternative in each constraint.
    for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
         cIndex != eIndex; ++cIndex) {
      AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
      if (cInfo.Type == InlineAsm::isClobber)
        continue;
      cInfo.selectAlternative(bestMAIndex);
    }
  }
}

// Check and hook up tied operands, choose constraint code to use.
for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
     cIndex != eIndex; ++cIndex) {
  AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];

  // If this is an output operand with a matching input operand, look up the
  // matching input. If their types mismatch, e.g. one is an integer, the
  // other is floating point, or their sizes are different, flag it as an
  // error.
  if (OpInfo.hasMatchingInput()) {
    AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];

    if (OpInfo.ConstraintVT != Input.ConstraintVT) {
      std::pair<unsigned, const TargetRegisterClass *> MatchRC =
          getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
                                       OpInfo.ConstraintVT);
      std::pair<unsigned, const TargetRegisterClass *> InputRC =
          getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
                                       Input.ConstraintVT);
      if ((OpInfo.ConstraintVT.isInteger() !=
           Input.ConstraintVT.isInteger()) ||
          (MatchRC.second != InputRC.second)) {
        report_fatal_error("Unsupported asm: input constraint"
                           " with a matching output constraint of"
                           " incompatible type!");
      }
    }
  }
}

return ConstraintOperands;
4362}

4364/// Return an integer indicating how general CT is.
4365static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
switch (CT) {
case TargetLowering::C_Immediate:
case TargetLowering::C_Other:
case TargetLowering::C_Unknown:
  return 0;
case TargetLowering::C_Register:
  return 1;
case TargetLowering::C_RegisterClass:
  return 2;
case TargetLowering::C_Memory:
  return 3;
}
llvm_unreachable("Invalid constraint type")::llvm::llvm_unreachable_internal("Invalid constraint type", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4378);
4379}

4381/// Examine constraint type and operand type and determine a weight value.
4382/// This object must already have been set up with the operand type
4383/// and the current alternative constraint selected.
4384TargetLowering::ConstraintWeight
TargetLowering::getMultipleConstraintMatchWeight(
  AsmOperandInfo &info, int maIndex) const {
InlineAsm::ConstraintCodeVector *rCodes;
if (maIndex >= (int)info.multipleAlternatives.size())
  rCodes = &info.Codes;
else
  rCodes = &info.multipleAlternatives[maIndex].Codes;
ConstraintWeight BestWeight = CW_Invalid;

// Loop over the options, keeping track of the most general one.
for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
  ConstraintWeight weight =
    getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
  if (weight > BestWeight)
    BestWeight = weight;
}

return BestWeight;
4403}

4405/// Examine constraint type and operand type and determine a weight value.
4406/// This object must already have been set up with the operand type
4407/// and the current alternative constraint selected.
4408TargetLowering::ConstraintWeight
TargetLowering::getSingleConstraintMatchWeight(
  AsmOperandInfo &info, const char *constraint) const {
ConstraintWeight weight = CW_Invalid;
Value *CallOperandVal = info.CallOperandVal;
  // If we don't have a value, we can't do a match,
  // but allow it at the lowest weight.
if (!CallOperandVal)
  return CW_Default;
// Look at the constraint type.
switch (*constraint) {
  case 'i': // immediate integer.
  case 'n': // immediate integer with a known value.
    if (isa<ConstantInt>(CallOperandVal))
      weight = CW_Constant;
    break;
  case 's': // non-explicit intregal immediate.
    if (isa<GlobalValue>(CallOperandVal))
      weight = CW_Constant;
    break;
  case 'E': // immediate float if host format.
  case 'F': // immediate float.
    if (isa<ConstantFP>(CallOperandVal))
      weight = CW_Constant;
    break;
  case '<': // memory operand with autodecrement.
  case '>': // memory operand with autoincrement.
  case 'm': // memory operand.
  case 'o': // offsettable memory operand
  case 'V': // non-offsettable memory operand
    weight = CW_Memory;
    break;
  case 'r': // general register.
  case 'g': // general register, memory operand or immediate integer.
            // note: Clang converts "g" to "imr".
    if (CallOperandVal->getType()->isIntegerTy())
      weight = CW_Register;
    break;
  case 'X': // any operand.
default:
  weight = CW_Default;
  break;
}
return weight;
4452}

4454/// If there are multiple different constraints that we could pick for this
4455/// operand (e.g. "imr") try to pick the 'best' one.
4456/// This is somewhat tricky: constraints fall into four classes:
4457///    Other         -> immediates and magic values
4458///    Register      -> one specific register
4459///    RegisterClass -> a group of regs
4460///    Memory        -> memory
4461/// Ideally, we would pick the most specific constraint possible: if we have
4462/// something that fits into a register, we would pick it.  The problem here
4463/// is that if we have something that could either be in a register or in
4464/// memory that use of the register could cause selection of *other*
4465/// operands to fail: they might only succeed if we pick memory.  Because of
4466/// this the heuristic we use is:
4467///
4468///  1) If there is an 'other' constraint, and if the operand is valid for
4469///     that constraint, use it.  This makes us take advantage of 'i'
4470///     constraints when available.
4471///  2) Otherwise, pick the most general constraint present.  This prefers
4472///     'm' over 'r', for example.
4473///
4474static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
                           const TargetLowering &TLI,
                           SDValue Op, SelectionDAG *DAG) {
assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options")((OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes.size() > 1 && \"Doesn't have multiple constraint options\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4477, __PRETTY_FUNCTION__));
unsigned BestIdx = 0;
TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
int BestGenerality = -1;

// Loop over the options, keeping track of the most general one.
for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
  TargetLowering::ConstraintType CType =
    TLI.getConstraintType(OpInfo.Codes[i]);

  // If this is an 'other' or 'immediate' constraint, see if the operand is
  // valid for it. For example, on X86 we might have an 'rI' constraint. If
  // the operand is an integer in the range [0..31] we want to use I (saving a
  // load of a register), otherwise we must use 'r'.
  if ((CType == TargetLowering::C_Other ||
       CType == TargetLowering::C_Immediate) && Op.getNode()) {
    assert(OpInfo.Codes[i].size() == 1 &&((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4494, __PRETTY_FUNCTION__))
           "Unhandled multi-letter 'other' constraint")((OpInfo.Codes[i].size() == 1 && "Unhandled multi-letter 'other' constraint"
) ? static_cast<void> (0) : __assert_fail ("OpInfo.Codes[i].size() == 1 && \"Unhandled multi-letter 'other' constraint\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4494, __PRETTY_FUNCTION__));
    std::vector<SDValue> ResultOps;
    TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
                                     ResultOps, *DAG);
    if (!ResultOps.empty()) {
      BestType = CType;
      BestIdx = i;
      break;
    }
  }

  // Things with matching constraints can only be registers, per gcc
  // documentation.  This mainly affects "g" constraints.
  if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
    continue;

  // This constraint letter is more general than the previous one, use it.
  int Generality = getConstraintGenerality(CType);
  if (Generality > BestGenerality) {
    BestType = CType;
    BestIdx = i;
    BestGenerality = Generality;
  }
}

OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
OpInfo.ConstraintType = BestType;
4521}

4523/// Determines the constraint code and constraint type to use for the specific
4524/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
4525void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
                                          SDValue Op,
                                          SelectionDAG *DAG) const {
assert(!OpInfo.Codes.empty() && "Must have at least one constraint")((!OpInfo.Codes.empty() && "Must have at least one constraint"
) ? static_cast<void> (0) : __assert_fail ("!OpInfo.Codes.empty() && \"Must have at least one constraint\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4528, __PRETTY_FUNCTION__));

// Single-letter constraints ('r') are very common.
if (OpInfo.Codes.size() == 1) {
  OpInfo.ConstraintCode = OpInfo.Codes[0];
  OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
} else {
  ChooseConstraint(OpInfo, *this, Op, DAG);
}

// 'X' matches anything.
if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
  // Labels and constants are handled elsewhere ('X' is the only thing
  // that matches labels).  For Functions, the type here is the type of
  // the result, which is not what we want to look at; leave them alone.
  Value *v = OpInfo.CallOperandVal;
  if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
    OpInfo.CallOperandVal = v;
    return;
  }

  if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
    return;

  // Otherwise, try to resolve it to something we know about by looking at
  // the actual operand type.
  if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
    OpInfo.ConstraintCode = Repl;
    OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
  }
}
4559}

4561/// Given an exact SDIV by a constant, create a multiplication
4562/// with the multiplicative inverse of the constant.
4563static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
                            const SDLoc &dl, SelectionDAG &DAG,
                            SmallVectorImpl<SDNode *> &Created) {
SDValue Op0 = N->getOperand(0);
SDValue Op1 = N->getOperand(1);
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();

bool UseSRA = false;
SmallVector<SDValue, 16> Shifts, Factors;

auto BuildSDIVPattern = [&](ConstantSDNode *C) {
  if (C->isNullValue())
    return false;
  APInt Divisor = C->getAPIntValue();
  unsigned Shift = Divisor.countTrailingZeros();
  if (Shift) {
    Divisor.ashrInPlace(Shift);
    UseSRA = true;
  }
  // Calculate the multiplicative inverse, using Newton's method.
  APInt t;
  APInt Factor = Divisor;
  while ((t = Divisor * Factor) != 1)
    Factor *= APInt(Divisor.getBitWidth(), 2) - t;
  Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
  Factors.push_back(DAG.getConstant(Factor, dl, SVT));
  return true;
};

// Collect all magic values from the build vector.
if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
  return SDValue();

SDValue Shift, Factor;
if (VT.isVector()) {
  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
  Factor = DAG.getBuildVector(VT, dl, Factors);
} else {
  Shift = Shifts[0];
  Factor = Factors[0];
}

SDValue Res = Op0;

// Shift the value upfront if it is even, so the LSB is one.
if (UseSRA) {
  // TODO: For UDIV use SRL instead of SRA.
  SDNodeFlags Flags;
  Flags.setExact(true);
  Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
  Created.push_back(Res.getNode());
}

return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
4620}

4622SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
                            SelectionDAG &DAG,
                            SmallVectorImpl<SDNode *> &Created) const {
AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (TLI.isIntDivCheap(N->getValueType(0), Attr))
  return SDValue(N, 0); // Lower SDIV as SDIV
return SDValue();
4630}

4632/// Given an ISD::SDIV node expressing a divide by constant,
4633/// return a DAG expression to select that will generate the same value by
4634/// multiplying by a magic number.
4635/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4636SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
                                bool IsAfterLegalization,
                                SmallVectorImpl<SDNode *> &Created) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();
unsigned EltBits = VT.getScalarSizeInBits();

// Check to see if we can do this.
// FIXME: We should be more aggressive here.
if (!isTypeLegal(VT))
  return SDValue();

// If the sdiv has an 'exact' bit we can use a simpler lowering.
if (N->getFlags().hasExact())
  return BuildExactSDIV(*this, N, dl, DAG, Created);

SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;

auto BuildSDIVPattern = [&](ConstantSDNode *C) {
  if (C->isNullValue())
    return false;

  const APInt &Divisor = C->getAPIntValue();
  APInt::ms magics = Divisor.magic();
  int NumeratorFactor = 0;
  int ShiftMask = -1;

  if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
    // If d is +1/-1, we just multiply the numerator by +1/-1.
    NumeratorFactor = Divisor.getSExtValue();
    magics.m = 0;
    magics.s = 0;
    ShiftMask = 0;
  } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
    // If d > 0 and m < 0, add the numerator.
    NumeratorFactor = 1;
  } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
    // If d < 0 and m > 0, subtract the numerator.
    NumeratorFactor = -1;
  }

  MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
  Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
  Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
  ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
  return true;
};

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

// Collect the shifts / magic values from each element.
if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
  return SDValue();

SDValue MagicFactor, Factor, Shift, ShiftMask;
if (VT.isVector()) {
  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
  Factor = DAG.getBuildVector(VT, dl, Factors);
  Shift = DAG.getBuildVector(ShVT, dl, Shifts);
  ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
} else {
  MagicFactor = MagicFactors[0];
  Factor = Factors[0];
  Shift = Shifts[0];
  ShiftMask = ShiftMasks[0];
}

// Multiply the numerator (operand 0) by the magic value.
// FIXME: We should support doing a MUL in a wider type.
SDValue Q;
if (IsAfterLegalization ? isOperationLegal(ISD::MULHS, VT)
                        : isOperationLegalOrCustom(ISD::MULHS, VT))
  Q = DAG.getNode(ISD::MULHS, dl, VT, N0, MagicFactor);
else if (IsAfterLegalization ? isOperationLegal(ISD::SMUL_LOHI, VT)
                             : isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) {
  SDValue LoHi =
      DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), N0, MagicFactor);
  Q = SDValue(LoHi.getNode(), 1);
} else
  return SDValue(); // No mulhs or equivalent.
Created.push_back(Q.getNode());

// (Optionally) Add/subtract the numerator using Factor.
Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
Created.push_back(Factor.getNode());
Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
Created.push_back(Q.getNode());

// Shift right algebraic by shift value.
Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
Created.push_back(Q.getNode());

// Extract the sign bit, mask it and add it to the quotient.
SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
Created.push_back(T.getNode());
T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
Created.push_back(T.getNode());
return DAG.getNode(ISD::ADD, dl, VT, Q, T);
4739}

4741/// Given an ISD::UDIV node expressing a divide by constant,
4742/// return a DAG expression to select that will generate the same value by
4743/// multiplying by a magic number.
4744/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
4745SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
                                bool IsAfterLegalization,
                                SmallVectorImpl<SDNode *> &Created) const {
SDLoc dl(N);
EVT VT = N->getValueType(0);
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();
unsigned EltBits = VT.getScalarSizeInBits();

// Check to see if we can do this.
// FIXME: We should be more aggressive here.
if (!isTypeLegal(VT))
  return SDValue();

bool UseNPQ = false;
SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;

auto BuildUDIVPattern = [&](ConstantSDNode *C) {
  if (C->isNullValue())
    return false;
  // FIXME: We should use a narrower constant when the upper
  // bits are known to be zero.
  APInt Divisor = C->getAPIntValue();
  APInt::mu magics = Divisor.magicu();
  unsigned PreShift = 0, PostShift = 0;

  // If the divisor is even, we can avoid using the expensive fixup by
  // shifting the divided value upfront.
  if (magics.a != 0 && !Divisor[0]) {
    PreShift = Divisor.countTrailingZeros();
    // Get magic number for the shifted divisor.
    magics = Divisor.lshr(PreShift).magicu(PreShift);
    assert(magics.a == 0 && "Should use cheap fixup now")((magics.a == 0 && "Should use cheap fixup now") ? static_cast
<void> (0) : __assert_fail ("magics.a == 0 && \"Should use cheap fixup now\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4778, __PRETTY_FUNCTION__));
  }

  APInt Magic = magics.m;

  unsigned SelNPQ;
  if (magics.a == 0 || Divisor.isOneValue()) {
    assert(magics.s < Divisor.getBitWidth() &&((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4786, __PRETTY_FUNCTION__))
           "We shouldn't generate an undefined shift!")((magics.s < Divisor.getBitWidth() && "We shouldn't generate an undefined shift!"
) ? static_cast<void> (0) : __assert_fail ("magics.s < Divisor.getBitWidth() && \"We shouldn't generate an undefined shift!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4786, __PRETTY_FUNCTION__));
    PostShift = magics.s;
    SelNPQ = false;
  } else {
    PostShift = magics.s - 1;
    SelNPQ = true;
  }

  PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
  MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
  NPQFactors.push_back(
      DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
                             : APInt::getNullValue(EltBits),
                      dl, SVT));
  PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
  UseNPQ |= SelNPQ;
  return true;
};

SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);

// Collect the shifts/magic values from each element.
if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
  return SDValue();

SDValue PreShift, PostShift, MagicFactor, NPQFactor;
if (VT.isVector()) {
  PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
  MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
  NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
  PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
} else {
  PreShift = PreShifts[0];
  MagicFactor = MagicFactors[0];
  PostShift = PostShifts[0];
}

SDValue Q = N0;
Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
Created.push_back(Q.getNode());

// FIXME: We should support doing a MUL in a wider type.
auto GetMULHU = [&](SDValue X, SDValue Y) {
  if (IsAfterLegalization ? isOperationLegal(ISD::MULHU, VT)
                          : isOperationLegalOrCustom(ISD::MULHU, VT))
    return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
  if (IsAfterLegalization ? isOperationLegal(ISD::UMUL_LOHI, VT)
                          : isOperationLegalOrCustom(ISD::UMUL_LOHI, VT)) {
    SDValue LoHi =
        DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
    return SDValue(LoHi.getNode(), 1);
  }
  return SDValue(); // No mulhu or equivalent
};

// Multiply the numerator (operand 0) by the magic value.
Q = GetMULHU(Q, MagicFactor);
if (!Q)
  return SDValue();

Created.push_back(Q.getNode());

if (UseNPQ) {
  SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
  Created.push_back(NPQ.getNode());

  // For vectors we might have a mix of non-NPQ/NPQ paths, so use
  // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
  if (VT.isVector())
    NPQ = GetMULHU(NPQ, NPQFactor);
  else
    NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));

  Created.push_back(NPQ.getNode());

  Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
  Created.push_back(Q.getNode());
}

Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
Created.push_back(Q.getNode());

SDValue One = DAG.getConstant(1, dl, VT);
SDValue IsOne = DAG.getSetCC(dl, VT, N1, One, ISD::SETEQ);
return DAG.getSelect(dl, VT, IsOne, N0, Q);
4872}

4874/// If all values in Values that *don't* match the predicate are same 'splat'
4875/// value, then replace all values with that splat value.
4876/// Else, if AlternativeReplacement was provided, then replace all values that
4877/// do match predicate with AlternativeReplacement value.
4878static void
4879turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
                        std::function<bool(SDValue)> Predicate,
                        SDValue AlternativeReplacement = SDValue()) {
SDValue Replacement;
// Is there a value for which the Predicate does *NOT* match? What is it?
auto SplatValue = llvm::find_if_not(Values, Predicate);
if (SplatValue != Values.end()) {
  // Does Values consist only of SplatValue's and values matching Predicate?
  if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
        return Value == *SplatValue || Predicate(Value);
      })) // Then we shall replace values matching predicate with SplatValue.
    Replacement = *SplatValue;
}
if (!Replacement) {
  // Oops, we did not find the "baseline" splat value.
  if (!AlternativeReplacement)
    return; // Nothing to do.
  // Let's replace with provided value then.
  Replacement = AlternativeReplacement;
}
std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
4900}

4902/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
4903/// where the divisor is constant and the comparison target is zero,
4904/// return a DAG expression that will generate the same comparison result
4905/// using only multiplications, additions and shifts/rotations.
4906/// Ref: "Hacker's Delight" 10-17.
4907SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
                                      SDValue CompTargetNode,
                                      ISD::CondCode Cond,
                                      DAGCombinerInfo &DCI,
                                      const SDLoc &DL) const {
SmallVector<SDNode *, 2> Built;
if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
                                       DCI, DL, Built)) {
  for (SDNode *N : Built)
    DCI.AddToWorklist(N);
  return Folded;
}

return SDValue();
4921}

4923SDValue
4924TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
                                SDValue CompTargetNode, ISD::CondCode Cond,
                                DAGCombinerInfo &DCI, const SDLoc &DL,
                                SmallVectorImpl<SDNode *> &Created) const {
// fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
// - D must be constant, with D = D0 * 2^K where D0 is odd
// - P is the multiplicative inverse of D0 modulo 2^W
// - Q = floor(((2^W) - 1) / D)
// where W is the width of the common type of N and D.
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4934, __PRETTY_FUNCTION__))
       "Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4934, __PRETTY_FUNCTION__));

SelectionDAG &DAG = DCI.DAG;

EVT VT = REMNode.getValueType();
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();

// If MUL is unavailable, we cannot proceed in any case.
if (!isOperationLegalOrCustom(ISD::MUL, VT))
  return SDValue();

// TODO: Could support comparing with non-zero too.
ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
if (!CompTarget || !CompTarget->isNullValue())
  return SDValue();

bool HadOneDivisor = false;
bool AllDivisorsAreOnes = true;
bool HadEvenDivisor = false;
bool AllDivisorsArePowerOfTwo = true;
SmallVector<SDValue, 16> PAmts, KAmts, QAmts;

auto BuildUREMPattern = [&](ConstantSDNode *C) {
  // Division by 0 is UB. Leave it to be constant-folded elsewhere.
  if (C->isNullValue())
    return false;

  const APInt &D = C->getAPIntValue();
  // If all divisors are ones, we will prefer to avoid the fold.
  HadOneDivisor |= D.isOneValue();
  AllDivisorsAreOnes &= D.isOneValue();

  // Decompose D into D0 * 2^K
  unsigned K = D.countTrailingZeros();
  assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4970, __PRETTY_FUNCTION__));
  APInt D0 = D.lshr(K);

  // D is even if it has trailing zeros.
  HadEvenDivisor |= (K != 0);
  // D is a power-of-two if D0 is one.
  // If all divisors are power-of-two, we will prefer to avoid the fold.
  AllDivisorsArePowerOfTwo &= D0.isOneValue();

  // P = inv(D0, 2^W)
  // 2^W requires W + 1 bits, so we have to extend and then truncate.
  unsigned W = D.getBitWidth();
  APInt P = D0.zext(W + 1)
                .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
                .trunc(W);
  assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
 static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4985, __PRETTY_FUNCTION__)); // unreachable
  assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4986, __PRETTY_FUNCTION__));

  // Q = floor((2^W - 1) / D)
  APInt Q = APInt::getAllOnesValue(W).udiv(D);

  assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
 "We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4992, __PRETTY_FUNCTION__))
         "We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
 "We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 4992, __PRETTY_FUNCTION__));

  // If the divisor is 1 the result can be constant-folded.
  if (D.isOneValue()) {
    // Set P and K amount to a bogus values so we can try to splat them.
    P = 0;
    K = -1;
    assert(Q.isAllOnesValue() &&((Q.isAllOnesValue() && "Expecting all-ones comparison for one divisor"
) ? static_cast<void> (0) : __assert_fail ("Q.isAllOnesValue() && \"Expecting all-ones comparison for one divisor\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5000, __PRETTY_FUNCTION__))
           "Expecting all-ones comparison for one divisor")((Q.isAllOnesValue() && "Expecting all-ones comparison for one divisor"
) ? static_cast<void> (0) : __assert_fail ("Q.isAllOnesValue() && \"Expecting all-ones comparison for one divisor\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5000, __PRETTY_FUNCTION__));
  }

  PAmts.push_back(DAG.getConstant(P, DL, SVT));
  KAmts.push_back(
      DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
  QAmts.push_back(DAG.getConstant(Q, DL, SVT));
  return true;
};

SDValue N = REMNode.getOperand(0);
SDValue D = REMNode.getOperand(1);

// Collect the values from each element.
if (!ISD::matchUnaryPredicate(D, BuildUREMPattern))
  return SDValue();

// If this is a urem by a one, avoid the fold since it can be constant-folded.
if (AllDivisorsAreOnes)
  return SDValue();

// If this is a urem by a powers-of-two, avoid the fold since it can be
// best implemented as a bit test.
if (AllDivisorsArePowerOfTwo)
  return SDValue();

SDValue PVal, KVal, QVal;
if (VT.isVector()) {
  if (HadOneDivisor) {
    // Try to turn PAmts into a splat, since we don't care about the values
    // that are currently '0'. If we can't, just keep '0'`s.
    turnVectorIntoSplatVector(PAmts, isNullConstant);
    // Try to turn KAmts into a splat, since we don't care about the values
    // that are currently '-1'. If we can't, change them to '0'`s.
    turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
                              DAG.getConstant(0, DL, ShSVT));
  }

  PVal = DAG.getBuildVector(VT, DL, PAmts);
  KVal = DAG.getBuildVector(ShVT, DL, KAmts);
  QVal = DAG.getBuildVector(VT, DL, QAmts);
} else {
  PVal = PAmts[0];
  KVal = KAmts[0];
  QVal = QAmts[0];
}

// (mul N, P)
SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
Created.push_back(Op0.getNode());

// Rotate right only if any divisor was even. We avoid rotates for all-odd
// divisors as a performance improvement, since rotating by 0 is a no-op.
if (HadEvenDivisor) {
  // We need ROTR to do this.
  if (!isOperationLegalOrCustom(ISD::ROTR, VT))
    return SDValue();
  SDNodeFlags Flags;
  Flags.setExact(true);
  // UREM: (rotr (mul N, P), K)
  Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
  Created.push_back(Op0.getNode());
}

// UREM: (setule/setugt (rotr (mul N, P), K), Q)
return DAG.getSetCC(DL, SETCCVT, Op0, QVal,
                    ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5067}

5069/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5070/// where the divisor is constant and the comparison target is zero,
5071/// return a DAG expression that will generate the same comparison result
5072/// using only multiplications, additions and shifts/rotations.
5073/// Ref: "Hacker's Delight" 10-17.
5074SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
                                      SDValue CompTargetNode,
                                      ISD::CondCode Cond,
                                      DAGCombinerInfo &DCI,
                                      const SDLoc &DL) const {
SmallVector<SDNode *, 7> Built;
if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
                                       DCI, DL, Built)) {
  assert(Built.size() <= 7 && "Max size prediction failed.")((Built.size() <= 7 && "Max size prediction failed."
) ? static_cast<void> (0) : __assert_fail ("Built.size() <= 7 && \"Max size prediction failed.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5082, __PRETTY_FUNCTION__));
  for (SDNode *N : Built)
    DCI.AddToWorklist(N);
  return Folded;
}

return SDValue();
5089}

5091SDValue
5092TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
                                SDValue CompTargetNode, ISD::CondCode Cond,
                                DAGCombinerInfo &DCI, const SDLoc &DL,
                                SmallVectorImpl<SDNode *> &Created) const {
// Fold:
//   (seteq/ne (srem N, D), 0)
// To:
//   (setule/ugt (rotr (add (mul N, P), A), K), Q)
//
// - D must be constant, with D = D0 * 2^K where D0 is odd
// - P is the multiplicative inverse of D0 modulo 2^W
// - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
// - Q = floor((2 * A) / (2^K))
// where W is the width of the common type of N and D.
assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5107, __PRETTY_FUNCTION__))
       "Only applicable for (in)equality comparisons.")(((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Only applicable for (in)equality comparisons."
) ? static_cast<void> (0) : __assert_fail ("(Cond == ISD::SETEQ || Cond == ISD::SETNE) && \"Only applicable for (in)equality comparisons.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5107, __PRETTY_FUNCTION__));

SelectionDAG &DAG = DCI.DAG;

EVT VT = REMNode.getValueType();
EVT SVT = VT.getScalarType();
EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
EVT ShSVT = ShVT.getScalarType();

// If MUL is unavailable, we cannot proceed in any case.
if (!isOperationLegalOrCustom(ISD::MUL, VT))
  return SDValue();

// TODO: Could support comparing with non-zero too.
ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
if (!CompTarget || !CompTarget->isNullValue())
  return SDValue();

bool HadIntMinDivisor = false;
bool HadOneDivisor = false;
bool AllDivisorsAreOnes = true;
bool HadEvenDivisor = false;
bool NeedToApplyOffset = false;
bool AllDivisorsArePowerOfTwo = true;
SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;

auto BuildSREMPattern = [&](ConstantSDNode *C) {
  // Division by 0 is UB. Leave it to be constant-folded elsewhere.
  if (C->isNullValue())
1
Taking false branch→
    return false;

  // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.

  // WARNING: this fold is only valid for positive divisors!
  APInt D = C->getAPIntValue();
  if (D.isNegative())
2
←
Taking false branch→
    D.negate(); //  `rem %X, -C` is equivalent to `rem %X, C`

  HadIntMinDivisor |= D.isMinSignedValue();

  // If all divisors are ones, we will prefer to avoid the fold.
  HadOneDivisor |= D.isOneValue();
  AllDivisorsAreOnes &= D.isOneValue();

  // Decompose D into D0 * 2^K
  unsigned K = D.countTrailingZeros();
  assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.")(((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate."
) ? static_cast<void> (0) : __assert_fail ("(!D.isOneValue() || (K == 0)) && \"For divisor '1' we won't rotate.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5153, __PRETTY_FUNCTION__));
3
←
Assuming 'K' is equal to 0→
4
←
'?' condition is true→
  APInt D0 = D.lshr(K);

  if (!D.isMinSignedValue()) {
5
←
Calling 'APInt::isMinSignedValue'→
    // D is even if it has trailing zeros; unless it's INT_MIN, in which case
    // we don't care about this lane in this fold, we'll special-handle it.
    HadEvenDivisor |= (K != 0);
  }

  // D is a power-of-two if D0 is one. This includes INT_MIN.
  // If all divisors are power-of-two, we will prefer to avoid the fold.
  AllDivisorsArePowerOfTwo &= D0.isOneValue();

  // P = inv(D0, 2^W)
  // 2^W requires W + 1 bits, so we have to extend and then truncate.
  unsigned W = D.getBitWidth();
  APInt P = D0.zext(W + 1)
                .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
                .trunc(W);
  assert(!P.isNullValue() && "No multiplicative inverse!")((!P.isNullValue() && "No multiplicative inverse!") ?
 static_cast<void> (0) : __assert_fail ("!P.isNullValue() && \"No multiplicative inverse!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5172, __PRETTY_FUNCTION__)); // unreachable
  assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.")(((D0 * P).isOneValue() && "Multiplicative inverse sanity check."
) ? static_cast<void> (0) : __assert_fail ("(D0 * P).isOneValue() && \"Multiplicative inverse sanity check.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5173, __PRETTY_FUNCTION__));

  // A = floor((2^(W - 1) - 1) / D0) & -2^K
  APInt A = APInt::getSignedMaxValue(W).udiv(D0);
  A.clearLowBits(K);

  if (!D.isMinSignedValue()) {
    // If divisor INT_MIN, then we don't care about this lane in this fold,
    // we'll special-handle it.
    NeedToApplyOffset |= A != 0;
  }

  // Q = floor((2 * A) / (2^K))
  APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));

  assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&((APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
 "We are expecting that A is always less than all-ones for SVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5189, __PRETTY_FUNCTION__))
         "We are expecting that A is always less than all-ones for SVT")((APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
 "We are expecting that A is always less than all-ones for SVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) && \"We are expecting that A is always less than all-ones for SVT\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5189, __PRETTY_FUNCTION__));
  assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
 "We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5191, __PRETTY_FUNCTION__))
         "We are expecting that K is always less than all-ones for ShSVT")((APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
 "We are expecting that K is always less than all-ones for ShSVT"
) ? static_cast<void> (0) : __assert_fail ("APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) && \"We are expecting that K is always less than all-ones for ShSVT\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5191, __PRETTY_FUNCTION__));

  // If the divisor is 1 the result can be constant-folded. Likewise, we
  // don't care about INT_MIN lanes, those can be set to undef if appropriate.
  if (D.isOneValue()) {
    // Set P, A and K to a bogus values so we can try to splat them.
    P = 0;
    A = -1;
    K = -1;

    // x ?% 1 == 0  <-->  true  <-->  x u<= -1
    Q = -1;
  }

  PAmts.push_back(DAG.getConstant(P, DL, SVT));
  AAmts.push_back(DAG.getConstant(A, DL, SVT));
  KAmts.push_back(
      DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
  QAmts.push_back(DAG.getConstant(Q, DL, SVT));
  return true;
};

SDValue N = REMNode.getOperand(0);
SDValue D = REMNode.getOperand(1);

// Collect the values from each element.
if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
  return SDValue();

// If this is a srem by a one, avoid the fold since it can be constant-folded.
if (AllDivisorsAreOnes)
  return SDValue();

// If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
// since it can be best implemented as a bit test.
if (AllDivisorsArePowerOfTwo)
  return SDValue();

SDValue PVal, AVal, KVal, QVal;
if (VT.isVector()) {
  if (HadOneDivisor) {
    // Try to turn PAmts into a splat, since we don't care about the values
    // that are currently '0'. If we can't, just keep '0'`s.
    turnVectorIntoSplatVector(PAmts, isNullConstant);
    // Try to turn AAmts into a splat, since we don't care about the
    // values that are currently '-1'. If we can't, change them to '0'`s.
    turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
                              DAG.getConstant(0, DL, SVT));
    // Try to turn KAmts into a splat, since we don't care about the values
    // that are currently '-1'. If we can't, change them to '0'`s.
    turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
                              DAG.getConstant(0, DL, ShSVT));
  }

  PVal = DAG.getBuildVector(VT, DL, PAmts);
  AVal = DAG.getBuildVector(VT, DL, AAmts);
  KVal = DAG.getBuildVector(ShVT, DL, KAmts);
  QVal = DAG.getBuildVector(VT, DL, QAmts);
} else {
  PVal = PAmts[0];
  AVal = AAmts[0];
  KVal = KAmts[0];
  QVal = QAmts[0];
}

// (mul N, P)
SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
Created.push_back(Op0.getNode());

if (NeedToApplyOffset) {
  // We need ADD to do this.
  if (!isOperationLegalOrCustom(ISD::ADD, VT))
    return SDValue();

  // (add (mul N, P), A)
  Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
  Created.push_back(Op0.getNode());
}

// Rotate right only if any divisor was even. We avoid rotates for all-odd
// divisors as a performance improvement, since rotating by 0 is a no-op.
if (HadEvenDivisor) {
  // We need ROTR to do this.
  if (!isOperationLegalOrCustom(ISD::ROTR, VT))
    return SDValue();
  SDNodeFlags Flags;
  Flags.setExact(true);
  // SREM: (rotr (add (mul N, P), A), K)
  Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
  Created.push_back(Op0.getNode());
}

// SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
SDValue Fold =
    DAG.getSetCC(DL, SETCCVT, Op0, QVal,
                 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));

// If we didn't have lanes with INT_MIN divisor, then we're done.
if (!HadIntMinDivisor)
  return Fold;

// That fold is only valid for positive divisors. Which effectively means,
// it is invalid for INT_MIN divisors. So if we have such a lane,
// we must fix-up results for said lanes.
assert(VT.isVector() && "Can/should only get here for vectors.")((VT.isVector() && "Can/should only get here for vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can/should only get here for vectors.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5295, __PRETTY_FUNCTION__));

if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
    !isOperationLegalOrCustom(ISD::AND, VT) ||
    !isOperationLegalOrCustom(Cond, VT) ||
    !isOperationLegalOrCustom(ISD::VSELECT, VT))
  return SDValue();

Created.push_back(Fold.getNode());

SDValue IntMin = DAG.getConstant(
    APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
SDValue IntMax = DAG.getConstant(
    APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
SDValue Zero =
    DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);

// Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
Created.push_back(DivisorIsIntMin.getNode());

// (N s% INT_MIN) ==/!= 0  <-->  (N & INT_MAX) ==/!= 0
SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
Created.push_back(Masked.getNode());
SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
Created.push_back(MaskedIsZero.getNode());

// To produce final result we need to blend 2 vectors: 'SetCC' and
// 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
// from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
// constant-folded, select can get lowered to a shuffle with constant mask.
SDValue Blended =
    DAG.getNode(ISD::VSELECT, DL, VT, DivisorIsIntMin, MaskedIsZero, Fold);

return Blended;
5330}

5332bool TargetLowering::
5333verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
if (!isa<ConstantSDNode>(Op.getOperand(0))) {
  DAG.getContext()->emitError("argument to '__builtin_return_address' must "
                              "be a constant integer");
  return true;
}

return false;
5341}

5343char TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
                                      bool LegalOperations, bool ForCodeSize,
                                      unsigned Depth) const {
// fneg is removable even if it has multiple uses.
if (Op.getOpcode() == ISD::FNEG)
  return 2;

// Don't allow anything with multiple uses unless we know it is free.
EVT VT = Op.getValueType();
const SDNodeFlags Flags = Op->getFlags();
const TargetOptions &Options = DAG.getTarget().Options;
if (!Op.hasOneUse() && !(Op.getOpcode() == ISD::FP_EXTEND &&
                         isFPExtFree(VT, Op.getOperand(0).getValueType())))
  return 0;

// Don't recurse exponentially.
if (Depth > SelectionDAG::MaxRecursionDepth)
  return 0;

switch (Op.getOpcode()) {
case ISD::ConstantFP: {
  if (!LegalOperations)
    return 1;

  // Don't invert constant FP values after legalization unless the target says
  // the negated constant is legal.
  return isOperationLegal(ISD::ConstantFP, VT) ||
         isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
                      ForCodeSize);
}
case ISD::BUILD_VECTOR: {
  // Only permit BUILD_VECTOR of constants.
  if (llvm::any_of(Op->op_values(), [&](SDValue N) {
        return !N.isUndef() && !isa<ConstantFPSDNode>(N);
      }))
    return 0;
  if (!LegalOperations)
    return 1;
  if (isOperationLegal(ISD::ConstantFP, VT) &&
      isOperationLegal(ISD::BUILD_VECTOR, VT))
    return 1;
  return llvm::all_of(Op->op_values(), [&](SDValue N) {
    return N.isUndef() ||
           isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
                        ForCodeSize);
  });
}
case ISD::FADD:
  if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
    return 0;

  // After operation legalization, it might not be legal to create new FSUBs.
  if (LegalOperations && !isOperationLegalOrCustom(ISD::FSUB, VT))
    return 0;

  // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
  if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
                                  ForCodeSize, Depth + 1))
    return V;
  // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
  return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
                            ForCodeSize, Depth + 1);
case ISD::FSUB:
  // We can't turn -(A-B) into B-A when we honor signed zeros.
  if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
    return 0;

  // fold (fneg (fsub A, B)) -> (fsub B, A)
  return 1;

case ISD::FMUL:
case ISD::FDIV:
  // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
  if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
                                  ForCodeSize, Depth + 1))
    return V;

  // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
  if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
    if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
      return 0;

  return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
                            ForCodeSize, Depth + 1);

case ISD::FMA:
case ISD::FMAD: {
  if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
    return 0;

  // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
  // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
  char V2 = isNegatibleForFree(Op.getOperand(2), DAG, LegalOperations,
                               ForCodeSize, Depth + 1);
  if (!V2)
    return 0;

  // One of Op0/Op1 must be cheaply negatible, then select the cheapest.
  char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
                               ForCodeSize, Depth + 1);
  char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
                               ForCodeSize, Depth + 1);
  char V01 = std::max(V0, V1);
  return V01 ? std::max(V01, V2) : 0;
}

case ISD::FP_EXTEND:
case ISD::FP_ROUND:
case ISD::FSIN:
  return isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
                            ForCodeSize, Depth + 1);
}

return 0;
5457}

5459SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
                                           bool LegalOperations,
                                           bool ForCodeSize,
                                           unsigned Depth) const {
// fneg is removable even if it has multiple uses.
if (Op.getOpcode() == ISD::FNEG)
  return Op.getOperand(0);

assert(Depth <= SelectionDAG::MaxRecursionDepth &&((Depth <= SelectionDAG::MaxRecursionDepth && "getNegatedExpression doesn't match isNegatibleForFree"
) ? static_cast<void> (0) : __assert_fail ("Depth <= SelectionDAG::MaxRecursionDepth && \"getNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5468, __PRETTY_FUNCTION__))
       "getNegatedExpression doesn't match isNegatibleForFree")((Depth <= SelectionDAG::MaxRecursionDepth && "getNegatedExpression doesn't match isNegatibleForFree"
) ? static_cast<void> (0) : __assert_fail ("Depth <= SelectionDAG::MaxRecursionDepth && \"getNegatedExpression doesn't match isNegatibleForFree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5468, __PRETTY_FUNCTION__));
const SDNodeFlags Flags = Op->getFlags();

switch (Op.getOpcode()) {
case ISD::ConstantFP: {
  APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
  V.changeSign();
  return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
}
case ISD::BUILD_VECTOR: {
  SmallVector<SDValue, 4> Ops;
  for (SDValue C : Op->op_values()) {
    if (C.isUndef()) {
      Ops.push_back(C);
      continue;
    }
    APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
    V.changeSign();
    Ops.push_back(DAG.getConstantFP(V, SDLoc(Op), C.getValueType()));
  }
  return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops);
}
case ISD::FADD:
  assert((DAG.getTarget().Options.NoSignedZerosFPMath ||(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
 (0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5493, __PRETTY_FUNCTION__))
          Flags.hasNoSignedZeros()) &&(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
 (0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5493, __PRETTY_FUNCTION__))
         "Expected NSZ fp-flag")(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
 (0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5493, __PRETTY_FUNCTION__));

  // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
  if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
                         Depth + 1))
    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
                       getNegatedExpression(Op.getOperand(0), DAG,
                                            LegalOperations, ForCodeSize,
                                            Depth + 1),
                       Op.getOperand(1), Flags);
  // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
  return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
                     getNegatedExpression(Op.getOperand(1), DAG,
                                          LegalOperations, ForCodeSize,
                                          Depth + 1),
                     Op.getOperand(0), Flags);
case ISD::FSUB:
  // fold (fneg (fsub 0, B)) -> B
  if (ConstantFPSDNode *N0CFP =
          isConstOrConstSplatFP(Op.getOperand(0), /*AllowUndefs*/ true))
    if (N0CFP->isZero())
      return Op.getOperand(1);

  // fold (fneg (fsub A, B)) -> (fsub B, A)
  return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
                     Op.getOperand(1), Op.getOperand(0), Flags);

case ISD::FMUL:
case ISD::FDIV:
  // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
  if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
                         Depth + 1))
    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
                       getNegatedExpression(Op.getOperand(0), DAG,
                                            LegalOperations, ForCodeSize,
                                            Depth + 1),
                       Op.getOperand(1), Flags);

  // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
  return DAG.getNode(
      Op.getOpcode(), SDLoc(Op), Op.getValueType(), Op.getOperand(0),
      getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
                           ForCodeSize, Depth + 1),
      Flags);

case ISD::FMA:
case ISD::FMAD: {
  assert((DAG.getTarget().Options.NoSignedZerosFPMath ||(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
 (0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5542, __PRETTY_FUNCTION__))
          Flags.hasNoSignedZeros()) &&(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
 (0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5542, __PRETTY_FUNCTION__))
         "Expected NSZ fp-flag")(((DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros
()) && "Expected NSZ fp-flag") ? static_cast<void>
 (0) : __assert_fail ("(DAG.getTarget().Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) && \"Expected NSZ fp-flag\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5542, __PRETTY_FUNCTION__));

  SDValue Neg2 = getNegatedExpression(Op.getOperand(2), DAG, LegalOperations,
                                      ForCodeSize, Depth + 1);

  char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
                               ForCodeSize, Depth + 1);
  char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
                               ForCodeSize, Depth + 1);
  if (V0 >= V1) {
    // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
    SDValue Neg0 = getNegatedExpression(
        Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1);
    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Neg0,
                       Op.getOperand(1), Neg2, Flags);
  }

  // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
  SDValue Neg1 = getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
                                      ForCodeSize, Depth + 1);
  return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
                     Op.getOperand(0), Neg1, Neg2, Flags);
}

case ISD::FP_EXTEND:
case ISD::FSIN:
  return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
                     getNegatedExpression(Op.getOperand(0), DAG,
                                          LegalOperations, ForCodeSize,
                                          Depth + 1));
case ISD::FP_ROUND:
  return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
                     getNegatedExpression(Op.getOperand(0), DAG,
                                          LegalOperations, ForCodeSize,
                                          Depth + 1),
                     Op.getOperand(1));
}

llvm_unreachable("Unknown code")::llvm::llvm_unreachable_internal("Unknown code", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5580);
5581}

5583//===----------------------------------------------------------------------===//
5584// Legalization Utilities
5585//===----------------------------------------------------------------------===//

5587bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl,
                                  SDValue LHS, SDValue RHS,
                                  SmallVectorImpl<SDValue> &Result,
                                  EVT HiLoVT, SelectionDAG &DAG,
                                  MulExpansionKind Kind, SDValue LL,
                                  SDValue LH, SDValue RL, SDValue RH) const {
assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||((Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode ==
 ISD::SMUL_LOHI) ? static_cast<void> (0) : __assert_fail
 ("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5594, __PRETTY_FUNCTION__))
       Opcode == ISD::SMUL_LOHI)((Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode ==
 ISD::SMUL_LOHI) ? static_cast<void> (0) : __assert_fail
 ("Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI || Opcode == ISD::SMUL_LOHI"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5594, __PRETTY_FUNCTION__));

bool HasMULHS = (Kind == MulExpansionKind::Always) ||
                isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
bool HasMULHU = (Kind == MulExpansionKind::Always) ||
                isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
                    isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
                    isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);

if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
  return false;

unsigned OuterBitSize = VT.getScalarSizeInBits();
unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
unsigned LHSSB = DAG.ComputeNumSignBits(LHS);
unsigned RHSSB = DAG.ComputeNumSignBits(RHS);

// LL, LH, RL, and RH must be either all NULL or all set to a value.
assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||(((LL.getNode() && LH.getNode() && RL.getNode
() && RH.getNode()) || (!LL.getNode() && !LH.
getNode() && !RL.getNode() && !RH.getNode()))
 ? static_cast<void> (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5615, __PRETTY_FUNCTION__))
       (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()))(((LL.getNode() && LH.getNode() && RL.getNode
() && RH.getNode()) || (!LL.getNode() && !LH.
getNode() && !RL.getNode() && !RH.getNode()))
 ? static_cast<void> (0) : __assert_fail ("(LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) || (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode())"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp"
, 5615, __PRETTY_FUNCTION__));

SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
                        bool Signed) -> bool {
  if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
    Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
    Hi = SDValue(Lo.getNode(), 1);
    return true;
  }
  if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
    Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
    Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
    return true;
  }
  return false;
};

SDValue Lo, Hi;

if (!LL.getNode() && !RL.getNode() &&
    isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
  LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
  RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
}

if (!LL.getNode())
  return false;

APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
if (DAG.MaskedValueIsZero(LHS, HighMask) &&
    DAG.MaskedValueIsZero(RHS, HighMask)) {
  // The inputs are both zero-extended.
  if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
    Result.push_back(Lo);
    Result.push_back(Hi);
    if (Opcode != ISD::MUL) {
      SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
      Result.push_back(Zero);
      Result.push_back(Zero);
    }
    return true;
  }
}

if (!VT.isVector() && Opcode == ISD::MUL && LHSSB > InnerBitSize &&
    RHSSB > InnerBitSize) {
  // The input values are both sign-extended.
  // TODO non-MUL case?
  if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
    Result.push_back(Lo);
    Result.push_back(Hi);
    return true;
  }
}

unsigned ShiftAmount = OuterBitSize - InnerBitSize;
EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
  // FIXME getShiftAmountTy does not always return a sensible result when VT
  // is an illegal type, and so the type may be too small to fit the shift
  // amount. Override it with i32. The shift will have to be legalized.
  ShiftAmountTy = MVT::i32;
}
SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);

if (!LH.getNode() && !RH.getNode() &&
    isOperationLegalOrCustom(ISD::SRL, VT) &&
    isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
  LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
  LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
  RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
  RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
}

if (!LH.getNode())
  return false;

if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
  return false;

Result.push_back(Lo);

if (Opcode == ISD::MUL) {
  RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
  LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
  Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
  Result.push_back(Hi);
  return true;
}

// Compute the full width result.
auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
  Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
};

SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
  return false;

// This is effectively the add part of a multiply-add of half-sized operands,
// so it cannot overflow.
Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
  return false;

SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);

bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
                isOperationLegalOrCustom(ISD::ADDE, VT));
if (UseGlue)
  Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
                     Merge(Lo, Hi));
else
  Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
                     Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));

SDValue Carry = Next.getValue(1);
Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);

if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
  return false;

if (UseGlue)
  Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
                   Carry);
else
  Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
                   Zero, Carry);

Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));

if (Opcode == ISD::SMUL_LO