/build/source/llvm/include/llvm/IR/PatternMatch.h

Bug Summary

File:	build/source/llvm/include/llvm/IR/PatternMatch.h
Warning:	line 961, column 9 Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name InstCombineSelect.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/InstCombine -I /build/source/llvm/lib/Transforms/InstCombine -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp

/build/source/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp

→

1//===- InstCombineSelect.cpp ----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visitSelect function.
10//
11//===----------------------------------------------------------------------===//

13#include "InstCombineInternal.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/Analysis/AssumptionCache.h"
18#include "llvm/Analysis/CmpInstAnalysis.h"
19#include "llvm/Analysis/InstructionSimplify.h"
20#include "llvm/Analysis/OverflowInstAnalysis.h"
21#include "llvm/Analysis/ValueTracking.h"
22#include "llvm/Analysis/VectorUtils.h"
23#include "llvm/IR/BasicBlock.h"
24#include "llvm/IR/Constant.h"
25#include "llvm/IR/ConstantRange.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DerivedTypes.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/InstrTypes.h"
30#include "llvm/IR/Instruction.h"
31#include "llvm/IR/Instructions.h"
32#include "llvm/IR/IntrinsicInst.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/Operator.h"
35#include "llvm/IR/PatternMatch.h"
36#include "llvm/IR/Type.h"
37#include "llvm/IR/User.h"
38#include "llvm/IR/Value.h"
39#include "llvm/Support/Casting.h"
40#include "llvm/Support/ErrorHandling.h"
41#include "llvm/Support/KnownBits.h"
42#include "llvm/Transforms/InstCombine/InstCombiner.h"
43#include <cassert>
44#include <utility>

46#define DEBUG_TYPE"instcombine" "instcombine"
47#include "llvm/Transforms/Utils/InstructionWorklist.h"

49using namespace llvm;
50using namespace PatternMatch;


53/// Replace a select operand based on an equality comparison with the identity
54/// constant of a binop.
55static Instruction *foldSelectBinOpIdentity(SelectInst &Sel,
                                          const TargetLibraryInfo &TLI,
                                          InstCombinerImpl &IC) {
// The select condition must be an equality compare with a constant operand.
Value *X;
Constant *C;
CmpInst::Predicate Pred;
if (!match(Sel.getCondition(), m_Cmp(Pred, m_Value(X), m_Constant(C))))
  return nullptr;

bool IsEq;
if (ICmpInst::isEquality(Pred))
  IsEq = Pred == ICmpInst::ICMP_EQ;
else if (Pred == FCmpInst::FCMP_OEQ)
  IsEq = true;
else if (Pred == FCmpInst::FCMP_UNE)
  IsEq = false;
else
  return nullptr;

// A select operand must be a binop.
BinaryOperator *BO;
if (!match(Sel.getOperand(IsEq ? 1 : 2), m_BinOp(BO)))
  return nullptr;

// The compare constant must be the identity constant for that binop.
// If this a floating-point compare with 0.0, any zero constant will do.
Type *Ty = BO->getType();
Constant *IdC = ConstantExpr::getBinOpIdentity(BO->getOpcode(), Ty, true);
if (IdC != C) {
  if (!IdC || !CmpInst::isFPPredicate(Pred))
    return nullptr;
  if (!match(IdC, m_AnyZeroFP()) || !match(C, m_AnyZeroFP()))
    return nullptr;
}

// Last, match the compare variable operand with a binop operand.
Value *Y;
if (!BO->isCommutative() && !match(BO, m_BinOp(m_Value(Y), m_Specific(X))))
  return nullptr;
if (!match(BO, m_c_BinOp(m_Value(Y), m_Specific(X))))
  return nullptr;

// +0.0 compares equal to -0.0, and so it does not behave as required for this
// transform. Bail out if we can not exclude that possibility.
if (isa<FPMathOperator>(BO))
  if (!BO->hasNoSignedZeros() && !CannotBeNegativeZero(Y, &TLI))
    return nullptr;

// BO = binop Y, X
// S = { select (cmp eq X, C), BO, ? } or { select (cmp ne X, C), ?, BO }
// =>
// S = { select (cmp eq X, C),  Y, ? } or { select (cmp ne X, C), ?,  Y }
return IC.replaceOperand(Sel, IsEq ? 1 : 2, Y);
109}

111/// This folds:
112///  select (icmp eq (and X, C1)), TC, FC
113///    iff C1 is a power 2 and the difference between TC and FC is a power-of-2.
114/// To something like:
115///  (shr (and (X, C1)), (log2(C1) - log2(TC-FC))) + FC
116/// Or:
117///  (shl (and (X, C1)), (log2(TC-FC) - log2(C1))) + FC
118/// With some variations depending if FC is larger than TC, or the shift
119/// isn't needed, or the bit widths don't match.
120static Value *foldSelectICmpAnd(SelectInst &Sel, ICmpInst *Cmp,
                              InstCombiner::BuilderTy &Builder) {
const APInt *SelTC, *SelFC;
if (!match(Sel.getTrueValue(), m_APInt(SelTC)) ||
    !match(Sel.getFalseValue(), m_APInt(SelFC)))
  return nullptr;

// If this is a vector select, we need a vector compare.
Type *SelType = Sel.getType();
if (SelType->isVectorTy() != Cmp->getType()->isVectorTy())
  return nullptr;

Value *V;
APInt AndMask;
bool CreateAnd = false;
ICmpInst::Predicate Pred = Cmp->getPredicate();
if (ICmpInst::isEquality(Pred)) {
  if (!match(Cmp->getOperand(1), m_Zero()))
    return nullptr;

  V = Cmp->getOperand(0);
  const APInt *AndRHS;
  if (!match(V, m_And(m_Value(), m_Power2(AndRHS))))
    return nullptr;

  AndMask = *AndRHS;
} else if (decomposeBitTestICmp(Cmp->getOperand(0), Cmp->getOperand(1),
                                Pred, V, AndMask)) {
  assert(ICmpInst::isEquality(Pred) && "Not equality test?")(static_cast <bool> (ICmpInst::isEquality(Pred) &&
 "Not equality test?") ? void (0) : __assert_fail ("ICmpInst::isEquality(Pred) && \"Not equality test?\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 148
, __extension__ __PRETTY_FUNCTION__));
  if (!AndMask.isPowerOf2())
    return nullptr;

  CreateAnd = true;
} else {
  return nullptr;
}

// In general, when both constants are non-zero, we would need an offset to
// replace the select. This would require more instructions than we started
// with. But there's one special-case that we handle here because it can
// simplify/reduce the instructions.
APInt TC = *SelTC;
APInt FC = *SelFC;
if (!TC.isZero() && !FC.isZero()) {
  // If the select constants differ by exactly one bit and that's the same
  // bit that is masked and checked by the select condition, the select can
  // be replaced by bitwise logic to set/clear one bit of the constant result.
  if (TC.getBitWidth() != AndMask.getBitWidth() || (TC ^ FC) != AndMask)
    return nullptr;
  if (CreateAnd) {
    // If we have to create an 'and', then we must kill the cmp to not
    // increase the instruction count.
    if (!Cmp->hasOneUse())
      return nullptr;
    V = Builder.CreateAnd(V, ConstantInt::get(SelType, AndMask));
  }
  bool ExtraBitInTC = TC.ugt(FC);
  if (Pred == ICmpInst::ICMP_EQ) {
    // If the masked bit in V is clear, clear or set the bit in the result:
    // (V & AndMaskC) == 0 ? TC : FC --> (V & AndMaskC) ^ TC
    // (V & AndMaskC) == 0 ? TC : FC --> (V & AndMaskC) | TC
    Constant *C = ConstantInt::get(SelType, TC);
    return ExtraBitInTC ? Builder.CreateXor(V, C) : Builder.CreateOr(V, C);
  }
  if (Pred == ICmpInst::ICMP_NE) {
    // If the masked bit in V is set, set or clear the bit in the result:
    // (V & AndMaskC) != 0 ? TC : FC --> (V & AndMaskC) | FC
    // (V & AndMaskC) != 0 ? TC : FC --> (V & AndMaskC) ^ FC
    Constant *C = ConstantInt::get(SelType, FC);
    return ExtraBitInTC ? Builder.CreateOr(V, C) : Builder.CreateXor(V, C);
  }
  llvm_unreachable("Only expecting equality predicates")::llvm::llvm_unreachable_internal("Only expecting equality predicates"
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 191
);
}

// Make sure one of the select arms is a power-of-2.
if (!TC.isPowerOf2() && !FC.isPowerOf2())
  return nullptr;

// Determine which shift is needed to transform result of the 'and' into the
// desired result.
const APInt &ValC = !TC.isZero() ? TC : FC;
unsigned ValZeros = ValC.logBase2();
unsigned AndZeros = AndMask.logBase2();

// Insert the 'and' instruction on the input to the truncate.
if (CreateAnd)
  V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), AndMask));

// If types don't match, we can still convert the select by introducing a zext
// or a trunc of the 'and'.
if (ValZeros > AndZeros) {
  V = Builder.CreateZExtOrTrunc(V, SelType);
  V = Builder.CreateShl(V, ValZeros - AndZeros);
} else if (ValZeros < AndZeros) {
  V = Builder.CreateLShr(V, AndZeros - ValZeros);
  V = Builder.CreateZExtOrTrunc(V, SelType);
} else {
  V = Builder.CreateZExtOrTrunc(V, SelType);
}

// Okay, now we know that everything is set up, we just don't know whether we
// have a icmp_ne or icmp_eq and whether the true or false val is the zero.
bool ShouldNotVal = !TC.isZero();
ShouldNotVal ^= Pred == ICmpInst::ICMP_NE;
if (ShouldNotVal)
  V = Builder.CreateXor(V, ValC);

return V;
228}

230/// We want to turn code that looks like this:
231///   %C = or %A, %B
232///   %D = select %cond, %C, %A
233/// into:
234///   %C = select %cond, %B, 0
235///   %D = or %A, %C
236///
237/// Assuming that the specified instruction is an operand to the select, return
238/// a bitmask indicating which operands of this instruction are foldable if they
239/// equal the other incoming value of the select.
240static unsigned getSelectFoldableOperands(BinaryOperator *I) {
switch (I->getOpcode()) {
case Instruction::Add:
case Instruction::FAdd:
case Instruction::Mul:
case Instruction::FMul:
case Instruction::And:
case Instruction::Or:
case Instruction::Xor:
  return 3;              // Can fold through either operand.
case Instruction::Sub:   // Can only fold on the amount subtracted.
case Instruction::FSub:
case Instruction::FDiv:  // Can only fold on the divisor amount.
case Instruction::Shl:   // Can only fold on the shift amount.
case Instruction::LShr:
case Instruction::AShr:
  return 1;
default:
  return 0;              // Cannot fold
}
260}

262/// We have (select c, TI, FI), and we know that TI and FI have the same opcode.
263Instruction *InstCombinerImpl::foldSelectOpOp(SelectInst &SI, Instruction *TI,
                                            Instruction *FI) {
// Don't break up min/max patterns. The hasOneUse checks below prevent that
// for most cases, but vector min/max with bitcasts can be transformed. If the
// one-use restrictions are eased for other patterns, we still don't want to
// obfuscate min/max.
if ((match(&SI, m_SMin(m_Value(), m_Value())) ||
     match(&SI, m_SMax(m_Value(), m_Value())) ||
     match(&SI, m_UMin(m_Value(), m_Value())) ||
     match(&SI, m_UMax(m_Value(), m_Value()))))
  return nullptr;

// If this is a cast from the same type, merge.
Value *Cond = SI.getCondition();
Type *CondTy = Cond->getType();
if (TI->getNumOperands() == 1 && TI->isCast()) {
  Type *FIOpndTy = FI->getOperand(0)->getType();
  if (TI->getOperand(0)->getType() != FIOpndTy)
    return nullptr;

  // The select condition may be a vector. We may only change the operand
  // type if the vector width remains the same (and matches the condition).
  if (auto *CondVTy = dyn_cast<VectorType>(CondTy)) {
    if (!FIOpndTy->isVectorTy() ||
        CondVTy->getElementCount() !=
            cast<VectorType>(FIOpndTy)->getElementCount())
      return nullptr;

    // TODO: If the backend knew how to deal with casts better, we could
    // remove this limitation. For now, there's too much potential to create
    // worse codegen by promoting the select ahead of size-altering casts
    // (PR28160).
    //
    // Note that ValueTracking's matchSelectPattern() looks through casts
    // without checking 'hasOneUse' when it matches min/max patterns, so this
    // transform may end up happening anyway.
    if (TI->getOpcode() != Instruction::BitCast &&
        (!TI->hasOneUse() || !FI->hasOneUse()))
      return nullptr;
  } else if (!TI->hasOneUse() || !FI->hasOneUse()) {
    // TODO: The one-use restrictions for a scalar select could be eased if
    // the fold of a select in visitLoadInst() was enhanced to match a pattern
    // that includes a cast.
    return nullptr;
  }

  // Fold this by inserting a select from the input values.
  Value *NewSI =
      Builder.CreateSelect(Cond, TI->getOperand(0), FI->getOperand(0),
                           SI.getName() + ".v", &SI);
  return CastInst::Create(Instruction::CastOps(TI->getOpcode()), NewSI,
                          TI->getType());
}

Value *OtherOpT, *OtherOpF;
bool MatchIsOpZero;
auto getCommonOp = [&](Instruction *TI, Instruction *FI, bool Commute,
                       bool Swapped = false) -> Value * {
  assert(!(Commute && Swapped) &&(static_cast <bool> (!(Commute && Swapped) &&
 "Commute and Swapped can't set at the same time") ? void (0)
 : __assert_fail ("!(Commute && Swapped) && \"Commute and Swapped can't set at the same time\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 322
, __extension__ __PRETTY_FUNCTION__))
         "Commute and Swapped can't set at the same time")(static_cast <bool> (!(Commute && Swapped) &&
 "Commute and Swapped can't set at the same time") ? void (0)
 : __assert_fail ("!(Commute && Swapped) && \"Commute and Swapped can't set at the same time\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 322
, __extension__ __PRETTY_FUNCTION__));
  if (!Swapped) {
    if (TI->getOperand(0) == FI->getOperand(0)) {
      OtherOpT = TI->getOperand(1);
      OtherOpF = FI->getOperand(1);
      MatchIsOpZero = true;
      return TI->getOperand(0);
    } else if (TI->getOperand(1) == FI->getOperand(1)) {
      OtherOpT = TI->getOperand(0);
      OtherOpF = FI->getOperand(0);
      MatchIsOpZero = false;
      return TI->getOperand(1);
    }
  }

  if (!Commute && !Swapped)
    return nullptr;

  // If we are allowing commute or swap of operands, then
  // allow a cross-operand match. In that case, MatchIsOpZero
  // means that TI's operand 0 (FI's operand 1) is the common op.
  if (TI->getOperand(0) == FI->getOperand(1)) {
    OtherOpT = TI->getOperand(1);
    OtherOpF = FI->getOperand(0);
    MatchIsOpZero = true;
    return TI->getOperand(0);
  } else if (TI->getOperand(1) == FI->getOperand(0)) {
    OtherOpT = TI->getOperand(0);
    OtherOpF = FI->getOperand(1);
    MatchIsOpZero = false;
    return TI->getOperand(1);
  }
  return nullptr;
};

if (TI->hasOneUse() || FI->hasOneUse()) {
  // Cond ? -X : -Y --> -(Cond ? X : Y)
  Value *X, *Y;
  if (match(TI, m_FNeg(m_Value(X))) && match(FI, m_FNeg(m_Value(Y)))) {
    // Intersect FMF from the fneg instructions and union those with the
    // select.
    FastMathFlags FMF = TI->getFastMathFlags();
    FMF &= FI->getFastMathFlags();
    FMF |= SI.getFastMathFlags();
    Value *NewSel =
        Builder.CreateSelect(Cond, X, Y, SI.getName() + ".v", &SI);
    if (auto *NewSelI = dyn_cast<Instruction>(NewSel))
      NewSelI->setFastMathFlags(FMF);
    Instruction *NewFNeg = UnaryOperator::CreateFNeg(NewSel);
    NewFNeg->setFastMathFlags(FMF);
    return NewFNeg;
  }

  // Min/max intrinsic with a common operand can have the common operand
  // pulled after the select. This is the same transform as below for binops,
  // but specialized for intrinsic matching and without the restrictive uses
  // clause.
  auto *TII = dyn_cast<IntrinsicInst>(TI);
  auto *FII = dyn_cast<IntrinsicInst>(FI);
  if (TII && FII && TII->getIntrinsicID() == FII->getIntrinsicID()) {
    if (match(TII, m_MaxOrMin(m_Value(), m_Value()))) {
      if (Value *MatchOp = getCommonOp(TI, FI, true)) {
        Value *NewSel =
            Builder.CreateSelect(Cond, OtherOpT, OtherOpF, "minmaxop", &SI);
        return CallInst::Create(TII->getCalledFunction(), {NewSel, MatchOp});
      }
    }
  }

  // icmp with a common operand also can have the common operand
  // pulled after the select.
  ICmpInst::Predicate TPred, FPred;
  if (match(TI, m_ICmp(TPred, m_Value(), m_Value())) &&
      match(FI, m_ICmp(FPred, m_Value(), m_Value()))) {
    if (TPred == FPred || TPred == CmpInst::getSwappedPredicate(FPred)) {
      bool Swapped = TPred != FPred;
      if (Value *MatchOp =
              getCommonOp(TI, FI, ICmpInst::isEquality(TPred), Swapped)) {
        Value *NewSel = Builder.CreateSelect(Cond, OtherOpT, OtherOpF,
                                             SI.getName() + ".v", &SI);
        return new ICmpInst(
            MatchIsOpZero ? TPred : CmpInst::getSwappedPredicate(TPred),
            MatchOp, NewSel);
      }
    }
  }
}

// Only handle binary operators (including two-operand getelementptr) with
// one-use here. As with the cast case above, it may be possible to relax the
// one-use constraint, but that needs be examined carefully since it may not
// reduce the total number of instructions.
if (TI->getNumOperands() != 2 || FI->getNumOperands() != 2 ||
    !TI->isSameOperationAs(FI) ||
    (!isa<BinaryOperator>(TI) && !isa<GetElementPtrInst>(TI)) ||
    !TI->hasOneUse() || !FI->hasOneUse())
  return nullptr;

// Figure out if the operations have any operands in common.
Value *MatchOp = getCommonOp(TI, FI, TI->isCommutative());
if (!MatchOp)
  return nullptr;

// If the select condition is a vector, the operands of the original select's
// operands also must be vectors. This may not be the case for getelementptr
// for example.
if (CondTy->isVectorTy() && (!OtherOpT->getType()->isVectorTy() ||
                             !OtherOpF->getType()->isVectorTy()))
  return nullptr;

// If we are sinking div/rem after a select, we may need to freeze the
// condition because div/rem may induce immediate UB with a poison operand.
// For example, the following transform is not safe if Cond can ever be poison
// because we can replace poison with zero and then we have div-by-zero that
// didn't exist in the original code:
// Cond ? x/y : x/z --> x / (Cond ? y : z)
auto *BO = dyn_cast<BinaryOperator>(TI);
if (BO && BO->isIntDivRem() && !isGuaranteedNotToBePoison(Cond)) {
  // A udiv/urem with a common divisor is safe because UB can only occur with
  // div-by-zero, and that would be present in the original code.
  if (BO->getOpcode() == Instruction::SDiv ||
      BO->getOpcode() == Instruction::SRem || MatchIsOpZero)
    Cond = Builder.CreateFreeze(Cond);
}

// If we reach here, they do have operations in common.
Value *NewSI = Builder.CreateSelect(Cond, OtherOpT, OtherOpF,
                                    SI.getName() + ".v", &SI);
Value *Op0 = MatchIsOpZero ? MatchOp : NewSI;
Value *Op1 = MatchIsOpZero ? NewSI : MatchOp;
if (auto *BO = dyn_cast<BinaryOperator>(TI)) {
  BinaryOperator *NewBO = BinaryOperator::Create(BO->getOpcode(), Op0, Op1);
  NewBO->copyIRFlags(TI);
  NewBO->andIRFlags(FI);
  return NewBO;
}
if (auto *TGEP = dyn_cast<GetElementPtrInst>(TI)) {
  auto *FGEP = cast<GetElementPtrInst>(FI);
  Type *ElementType = TGEP->getResultElementType();
  return TGEP->isInBounds() && FGEP->isInBounds()
             ? GetElementPtrInst::CreateInBounds(ElementType, Op0, {Op1})
             : GetElementPtrInst::Create(ElementType, Op0, {Op1});
}
llvm_unreachable("Expected BinaryOperator or GEP")::llvm::llvm_unreachable_internal("Expected BinaryOperator or GEP"
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 465
);
return nullptr;
467}

469static bool isSelect01(const APInt &C1I, const APInt &C2I) {
if (!C1I.isZero() && !C2I.isZero()) // One side must be zero.
  return false;
return C1I.isOne() || C1I.isAllOnes() || C2I.isOne() || C2I.isAllOnes();
473}

475/// Try to fold the select into one of the operands to allow further
476/// optimization.
477Instruction *InstCombinerImpl::foldSelectIntoOp(SelectInst &SI, Value *TrueVal,
                                              Value *FalseVal) {
// See the comment above GetSelectFoldableOperands for a description of the
// transformation we are doing here.
auto TryFoldSelectIntoOp = [&](SelectInst &SI, Value *TrueVal,
                               Value *FalseVal,
                               bool Swapped) -> Instruction * {
  auto *TVI = dyn_cast<BinaryOperator>(TrueVal);
  if (!TVI || !TVI->hasOneUse() || isa<Constant>(FalseVal))
    return nullptr;

  unsigned SFO = getSelectFoldableOperands(TVI);
  unsigned OpToFold = 0;
  if ((SFO & 1) && FalseVal == TVI->getOperand(0))
    OpToFold = 1;
  else if ((SFO & 2) && FalseVal == TVI->getOperand(1))
    OpToFold = 2;

  if (!OpToFold)
    return nullptr;

  // TODO: We probably ought to revisit cases where the select and FP
  // instructions have different flags and add tests to ensure the
  // behaviour is correct.
  FastMathFlags FMF;
  if (isa<FPMathOperator>(&SI))
    FMF = SI.getFastMathFlags();
  Constant *C = ConstantExpr::getBinOpIdentity(
      TVI->getOpcode(), TVI->getType(), true, FMF.noSignedZeros());
  Value *OOp = TVI->getOperand(2 - OpToFold);
  // Avoid creating select between 2 constants unless it's selecting
  // between 0, 1 and -1.
  const APInt *OOpC;
  bool OOpIsAPInt = match(OOp, m_APInt(OOpC));
  if (!isa<Constant>(OOp) ||
      (OOpIsAPInt && isSelect01(C->getUniqueInteger(), *OOpC))) {
    Value *NewSel = Builder.CreateSelect(SI.getCondition(), Swapped ? C : OOp,
                                         Swapped ? OOp : C);
    if (isa<FPMathOperator>(&SI))
      cast<Instruction>(NewSel)->setFastMathFlags(FMF);
    NewSel->takeName(TVI);
    BinaryOperator *BO =
        BinaryOperator::Create(TVI->getOpcode(), FalseVal, NewSel);
    BO->copyIRFlags(TVI);
    return BO;
  }
  return nullptr;
};

if (Instruction *R = TryFoldSelectIntoOp(SI, TrueVal, FalseVal, false))
  return R;

if (Instruction *R = TryFoldSelectIntoOp(SI, FalseVal, TrueVal, true))
  return R;

return nullptr;
533}

535/// We want to turn:
536///   (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1)
537/// into:
538///   zext (icmp ne i32 (and X, (or Y, (shl 1, Z))), 0)
539/// Note:
540///   Z may be 0 if lshr is missing.
541/// Worst-case scenario is that we will replace 5 instructions with 5 different
542/// instructions, but we got rid of select.
543static Instruction *foldSelectICmpAndAnd(Type *SelType, const ICmpInst *Cmp,
                                       Value *TVal, Value *FVal,
                                       InstCombiner::BuilderTy &Builder) {
if (!(Cmp->hasOneUse() && Cmp->getOperand(0)->hasOneUse() &&
      Cmp->getPredicate() == ICmpInst::ICMP_EQ &&
      match(Cmp->getOperand(1), m_Zero()) && match(FVal, m_One())))
  return nullptr;

// The TrueVal has general form of:  and %B, 1
Value *B;
if (!match(TVal, m_OneUse(m_And(m_Value(B), m_One()))))
  return nullptr;

// Where %B may be optionally shifted:  lshr %X, %Z.
Value *X, *Z;
const bool HasShift = match(B, m_OneUse(m_LShr(m_Value(X), m_Value(Z))));

// The shift must be valid.
// TODO: This restricts the fold to constant shift amounts. Is there a way to
//       handle variable shifts safely? PR47012
if (HasShift &&
    !match(Z, m_SpecificInt_ICMP(CmpInst::ICMP_ULT,
                                 APInt(SelType->getScalarSizeInBits(),
                                       SelType->getScalarSizeInBits()))))
  return nullptr;

if (!HasShift)
  X = B;

Value *Y;
if (!match(Cmp->getOperand(0), m_c_And(m_Specific(X), m_Value(Y))))
  return nullptr;

// ((X & Y) == 0) ? ((X >> Z) & 1) : 1 --> (X & (Y | (1 << Z))) != 0
// ((X & Y) == 0) ? (X & 1) : 1 --> (X & (Y | 1)) != 0
Constant *One = ConstantInt::get(SelType, 1);
Value *MaskB = HasShift ? Builder.CreateShl(One, Z) : One;
Value *FullMask = Builder.CreateOr(Y, MaskB);
Value *MaskedX = Builder.CreateAnd(X, FullMask);
Value *ICmpNeZero = Builder.CreateIsNotNull(MaskedX);
return new ZExtInst(ICmpNeZero, SelType);
584}

586/// We want to turn:
587///   (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1
588///   (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0
589/// into:
590///   ashr (X, Y)
591static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal,
                                   Value *FalseVal,
                                   InstCombiner::BuilderTy &Builder) {
ICmpInst::Predicate Pred = IC->getPredicate();
Value *CmpLHS = IC->getOperand(0);
Value *CmpRHS = IC->getOperand(1);
if (!CmpRHS->getType()->isIntOrIntVectorTy())
  return nullptr;

Value *X, *Y;
unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits();
if ((Pred != ICmpInst::ICMP_SGT ||
     !match(CmpRHS,
            m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, -1)))) &&
    (Pred != ICmpInst::ICMP_SLT ||
     !match(CmpRHS,
            m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, 0)))))
  return nullptr;

// Canonicalize so that ashr is in FalseVal.
if (Pred == ICmpInst::ICMP_SLT)
  std::swap(TrueVal, FalseVal);

if (match(TrueVal, m_LShr(m_Value(X), m_Value(Y))) &&
    match(FalseVal, m_AShr(m_Specific(X), m_Specific(Y))) &&
    match(CmpLHS, m_Specific(X))) {
  const auto *Ashr = cast<Instruction>(FalseVal);
  // if lshr is not exact and ashr is, this new ashr must not be exact.
  bool IsExact = Ashr->isExact() && cast<Instruction>(TrueVal)->isExact();
  return Builder.CreateAShr(X, Y, IC->getName(), IsExact);
}

return nullptr;
624}

626/// We want to turn:
627///   (select (icmp eq (and X, C1), 0), Y, (or Y, C2))
628/// into:
629///   (or (shl (and X, C1), C3), Y)
630/// iff:
631///   C1 and C2 are both powers of 2
632/// where:
633///   C3 = Log(C2) - Log(C1)
634///
635/// This transform handles cases where:
636/// 1. The icmp predicate is inverted
637/// 2. The select operands are reversed
638/// 3. The magnitude of C2 and C1 are flipped
639static Value *foldSelectICmpAndOr(const ICmpInst *IC, Value *TrueVal,
                                Value *FalseVal,
                                InstCombiner::BuilderTy &Builder) {
// Only handle integer compares. Also, if this is a vector select, we need a
// vector compare.
if (!TrueVal->getType()->isIntOrIntVectorTy() ||
    TrueVal->getType()->isVectorTy() != IC->getType()->isVectorTy())
  return nullptr;

Value *CmpLHS = IC->getOperand(0);
Value *CmpRHS = IC->getOperand(1);

Value *V;
unsigned C1Log;
bool IsEqualZero;
bool NeedAnd = false;
if (IC->isEquality()) {
  if (!match(CmpRHS, m_Zero()))
    return nullptr;

  const APInt *C1;
  if (!match(CmpLHS, m_And(m_Value(), m_Power2(C1))))
    return nullptr;

  V = CmpLHS;
  C1Log = C1->logBase2();
  IsEqualZero = IC->getPredicate() == ICmpInst::ICMP_EQ;
} else if (IC->getPredicate() == ICmpInst::ICMP_SLT ||
           IC->getPredicate() == ICmpInst::ICMP_SGT) {
  // We also need to recognize (icmp slt (trunc (X)), 0) and
  // (icmp sgt (trunc (X)), -1).
  IsEqualZero = IC->getPredicate() == ICmpInst::ICMP_SGT;
  if ((IsEqualZero && !match(CmpRHS, m_AllOnes())) ||
      (!IsEqualZero && !match(CmpRHS, m_Zero())))
    return nullptr;

  if (!match(CmpLHS, m_OneUse(m_Trunc(m_Value(V)))))
    return nullptr;

  C1Log = CmpLHS->getType()->getScalarSizeInBits() - 1;
  NeedAnd = true;
} else {
  return nullptr;
}

const APInt *C2;
bool OrOnTrueVal = false;
bool OrOnFalseVal = match(FalseVal, m_Or(m_Specific(TrueVal), m_Power2(C2)));
if (!OrOnFalseVal)
  OrOnTrueVal = match(TrueVal, m_Or(m_Specific(FalseVal), m_Power2(C2)));

if (!OrOnFalseVal && !OrOnTrueVal)
  return nullptr;

Value *Y = OrOnFalseVal ? TrueVal : FalseVal;

unsigned C2Log = C2->logBase2();

bool NeedXor = (!IsEqualZero && OrOnFalseVal) || (IsEqualZero && OrOnTrueVal);
bool NeedShift = C1Log != C2Log;
bool NeedZExtTrunc = Y->getType()->getScalarSizeInBits() !=
                     V->getType()->getScalarSizeInBits();

// Make sure we don't create more instructions than we save.
Value *Or = OrOnFalseVal ? FalseVal : TrueVal;
if ((NeedShift + NeedXor + NeedZExtTrunc) >
    (IC->hasOneUse() + Or->hasOneUse()))
  return nullptr;

if (NeedAnd) {
  // Insert the AND instruction on the input to the truncate.
  APInt C1 = APInt::getOneBitSet(V->getType()->getScalarSizeInBits(), C1Log);
  V = Builder.CreateAnd(V, ConstantInt::get(V->getType(), C1));
}

if (C2Log > C1Log) {
  V = Builder.CreateZExtOrTrunc(V, Y->getType());
  V = Builder.CreateShl(V, C2Log - C1Log);
} else if (C1Log > C2Log) {
  V = Builder.CreateLShr(V, C1Log - C2Log);
  V = Builder.CreateZExtOrTrunc(V, Y->getType());
} else
  V = Builder.CreateZExtOrTrunc(V, Y->getType());

if (NeedXor)
  V = Builder.CreateXor(V, *C2);

return Builder.CreateOr(V, Y);
727}

729/// Canonicalize a set or clear of a masked set of constant bits to
730/// select-of-constants form.
731static Instruction *foldSetClearBits(SelectInst &Sel,
                                   InstCombiner::BuilderTy &Builder) {
Value *Cond = Sel.getCondition();
Value *T = Sel.getTrueValue();
Value *F = Sel.getFalseValue();
Type *Ty = Sel.getType();
Value *X;
const APInt *NotC, *C;

// Cond ? (X & ~C) : (X | C) --> (X & ~C) | (Cond ? 0 : C)
if (match(T, m_And(m_Value(X), m_APInt(NotC))) &&
    match(F, m_OneUse(m_Or(m_Specific(X), m_APInt(C)))) && *NotC == ~(*C)) {
  Constant *Zero = ConstantInt::getNullValue(Ty);
  Constant *OrC = ConstantInt::get(Ty, *C);
  Value *NewSel = Builder.CreateSelect(Cond, Zero, OrC, "masksel", &Sel);
  return BinaryOperator::CreateOr(T, NewSel);
}

// Cond ? (X | C) : (X & ~C) --> (X & ~C) | (Cond ? C : 0)
if (match(F, m_And(m_Value(X), m_APInt(NotC))) &&
    match(T, m_OneUse(m_Or(m_Specific(X), m_APInt(C)))) && *NotC == ~(*C)) {
  Constant *Zero = ConstantInt::getNullValue(Ty);
  Constant *OrC = ConstantInt::get(Ty, *C);
  Value *NewSel = Builder.CreateSelect(Cond, OrC, Zero, "masksel", &Sel);
  return BinaryOperator::CreateOr(F, NewSel);
}

return nullptr;
759}

761//   select (x == 0), 0, x * y --> freeze(y) * x
762//   select (y == 0), 0, x * y --> freeze(x) * y
763//   select (x == 0), undef, x * y --> freeze(y) * x
764//   select (x == undef), 0, x * y --> freeze(y) * x
765// Usage of mul instead of 0 will make the result more poisonous,
766// so the operand that was not checked in the condition should be frozen.
767// The latter folding is applied only when a constant compared with x is
768// is a vector consisting of 0 and undefs. If a constant compared with x
769// is a scalar undefined value or undefined vector then an expression
770// should be already folded into a constant.
771static Instruction *foldSelectZeroOrMul(SelectInst &SI, InstCombinerImpl &IC) {
auto *CondVal = SI.getCondition();
auto *TrueVal = SI.getTrueValue();
auto *FalseVal = SI.getFalseValue();
Value *X, *Y;
ICmpInst::Predicate Predicate;

// Assuming that constant compared with zero is not undef (but it may be
// a vector with some undef elements). Otherwise (when a constant is undef)
// the select expression should be already simplified.
if (!match(CondVal, m_ICmp(Predicate, m_Value(X), m_Zero())) ||
    !ICmpInst::isEquality(Predicate))
  return nullptr;

if (Predicate == ICmpInst::ICMP_NE)
  std::swap(TrueVal, FalseVal);

// Check that TrueVal is a constant instead of matching it with m_Zero()
// to handle the case when it is a scalar undef value or a vector containing
// non-zero elements that are masked by undef elements in the compare
// constant.
auto *TrueValC = dyn_cast<Constant>(TrueVal);
if (TrueValC == nullptr ||
    !match(FalseVal, m_c_Mul(m_Specific(X), m_Value(Y))) ||
    !isa<Instruction>(FalseVal))
  return nullptr;

auto *ZeroC = cast<Constant>(cast<Instruction>(CondVal)->getOperand(1));
auto *MergedC = Constant::mergeUndefsWith(TrueValC, ZeroC);
// If X is compared with 0 then TrueVal could be either zero or undef.
// m_Zero match vectors containing some undef elements, but for scalars
// m_Undef should be used explicitly.
if (!match(MergedC, m_Zero()) && !match(MergedC, m_Undef()))
  return nullptr;

auto *FalseValI = cast<Instruction>(FalseVal);
auto *FrY = IC.InsertNewInstBefore(new FreezeInst(Y, Y->getName() + ".fr"),
                                   *FalseValI);
IC.replaceOperand(*FalseValI, FalseValI->getOperand(0) == Y ? 0 : 1, FrY);
return IC.replaceInstUsesWith(SI, FalseValI);
811}

813/// Transform patterns such as (a > b) ? a - b : 0 into usub.sat(a, b).
814/// There are 8 commuted/swapped variants of this pattern.
815/// TODO: Also support a - UMIN(a,b) patterns.
816static Value *canonicalizeSaturatedSubtract(const ICmpInst *ICI,
                                          const Value *TrueVal,
                                          const Value *FalseVal,
                                          InstCombiner::BuilderTy &Builder) {
ICmpInst::Predicate Pred = ICI->getPredicate();
Value *A = ICI->getOperand(0);
Value *B = ICI->getOperand(1);

// (b > a) ? 0 : a - b -> (b <= a) ? a - b : 0
// (a == 0) ? 0 : a - 1 -> (a != 0) ? a - 1 : 0
if (match(TrueVal, m_Zero())) {
  Pred = ICmpInst::getInversePredicate(Pred);
  std::swap(TrueVal, FalseVal);
}

if (!match(FalseVal, m_Zero()))
  return nullptr;

// ugt 0 is canonicalized to ne 0 and requires special handling
// (a != 0) ? a + -1 : 0 -> usub.sat(a, 1)
if (Pred == ICmpInst::ICMP_NE) {
  if (match(B, m_Zero()) && match(TrueVal, m_Add(m_Specific(A), m_AllOnes())))
    return Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, A,
                                         ConstantInt::get(A->getType(), 1));
  return nullptr;
}

if (!ICmpInst::isUnsigned(Pred))
  return nullptr;

if (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_ULT) {
  // (b < a) ? a - b : 0 -> (a > b) ? a - b : 0
  std::swap(A, B);
  Pred = ICmpInst::getSwappedPredicate(Pred);
}

assert((Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) &&(static_cast <bool> ((Pred == ICmpInst::ICMP_UGE || Pred
 == ICmpInst::ICMP_UGT) && "Unexpected isUnsigned predicate!"
) ? void (0) : __assert_fail ("(Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) && \"Unexpected isUnsigned predicate!\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 853
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected isUnsigned predicate!")(static_cast <bool> ((Pred == ICmpInst::ICMP_UGE || Pred
 == ICmpInst::ICMP_UGT) && "Unexpected isUnsigned predicate!"
) ? void (0) : __assert_fail ("(Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_UGT) && \"Unexpected isUnsigned predicate!\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 853
, __extension__ __PRETTY_FUNCTION__));

// Ensure the sub is of the form:
//  (a > b) ? a - b : 0 -> usub.sat(a, b)
//  (a > b) ? b - a : 0 -> -usub.sat(a, b)
// Checking for both a-b and a+(-b) as a constant.
bool IsNegative = false;
const APInt *C;
if (match(TrueVal, m_Sub(m_Specific(B), m_Specific(A))) ||
    (match(A, m_APInt(C)) &&
     match(TrueVal, m_Add(m_Specific(B), m_SpecificInt(-*C)))))
  IsNegative = true;
else if (!match(TrueVal, m_Sub(m_Specific(A), m_Specific(B))) &&
         !(match(B, m_APInt(C)) &&
           match(TrueVal, m_Add(m_Specific(A), m_SpecificInt(-*C)))))
  return nullptr;

// If we are adding a negate and the sub and icmp are used anywhere else, we
// would end up with more instructions.
if (IsNegative && !TrueVal->hasOneUse() && !ICI->hasOneUse())
  return nullptr;

// (a > b) ? a - b : 0 -> usub.sat(a, b)
// (a > b) ? b - a : 0 -> -usub.sat(a, b)
Value *Result = Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, A, B);
if (IsNegative)
  Result = Builder.CreateNeg(Result);
return Result;
881}

883static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
                                     InstCombiner::BuilderTy &Builder) {
if (!Cmp->hasOneUse())
  return nullptr;

// Match unsigned saturated add with constant.
Value *Cmp0 = Cmp->getOperand(0);
Value *Cmp1 = Cmp->getOperand(1);
ICmpInst::Predicate Pred = Cmp->getPredicate();
Value *X;
const APInt *C, *CmpC;
if (Pred == ICmpInst::ICMP_ULT &&
    match(TVal, m_Add(m_Value(X), m_APInt(C))) && X == Cmp0 &&
    match(FVal, m_AllOnes()) && match(Cmp1, m_APInt(CmpC)) && *CmpC == ~*C) {
  // (X u< ~C) ? (X + C) : -1 --> uadd.sat(X, C)
  return Builder.CreateBinaryIntrinsic(
      Intrinsic::uadd_sat, X, ConstantInt::get(X->getType(), *C));
}

// Match unsigned saturated add of 2 variables with an unnecessary 'not'.
// There are 8 commuted variants.
// Canonicalize -1 (saturated result) to true value of the select.
if (match(FVal, m_AllOnes())) {
  std::swap(TVal, FVal);
  Pred = CmpInst::getInversePredicate(Pred);
}
if (!match(TVal, m_AllOnes()))
  return nullptr;

// Canonicalize predicate to less-than or less-or-equal-than.
if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
  std::swap(Cmp0, Cmp1);
  Pred = CmpInst::getSwappedPredicate(Pred);
}
if (Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_ULE)
  return nullptr;

// Match unsigned saturated add of 2 variables with an unnecessary 'not'.
// Strictness of the comparison is irrelevant.
Value *Y;
if (match(Cmp0, m_Not(m_Value(X))) &&
    match(FVal, m_c_Add(m_Specific(X), m_Value(Y))) && Y == Cmp1) {
  // (~X u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
  // (~X u< Y) ? -1 : (Y + X) --> uadd.sat(X, Y)
  return Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, X, Y);
}
// The 'not' op may be included in the sum but not the compare.
// Strictness of the comparison is irrelevant.
X = Cmp0;
Y = Cmp1;
if (match(FVal, m_c_Add(m_Not(m_Specific(X)), m_Specific(Y)))) {
  // (X u< Y) ? -1 : (~X + Y) --> uadd.sat(~X, Y)
  // (X u< Y) ? -1 : (Y + ~X) --> uadd.sat(Y, ~X)
  BinaryOperator *BO = cast<BinaryOperator>(FVal);
  return Builder.CreateBinaryIntrinsic(
      Intrinsic::uadd_sat, BO->getOperand(0), BO->getOperand(1));
}
// The overflow may be detected via the add wrapping round.
// This is only valid for strict comparison!
if (Pred == ICmpInst::ICMP_ULT &&
    match(Cmp0, m_c_Add(m_Specific(Cmp1), m_Value(Y))) &&
    match(FVal, m_c_Add(m_Specific(Cmp1), m_Specific(Y)))) {
  // ((X + Y) u< X) ? -1 : (X + Y) --> uadd.sat(X, Y)
  // ((X + Y) u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
  return Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, Cmp1, Y);
}

return nullptr;
951}

953/// Try to match patterns with select and subtract as absolute difference.
954static Value *foldAbsDiff(ICmpInst *Cmp, Value *TVal, Value *FVal,
                        InstCombiner::BuilderTy &Builder) {
auto *TI = dyn_cast<Instruction>(TVal);
auto *FI = dyn_cast<Instruction>(FVal);
if (!TI || !FI)
  return nullptr;

// Normalize predicate to gt/lt rather than ge/le.
ICmpInst::Predicate Pred = Cmp->getStrictPredicate();
Value *A = Cmp->getOperand(0);
Value *B = Cmp->getOperand(1);

// Normalize "A - B" as the true value of the select.
if (match(FI, m_Sub(m_Specific(A), m_Specific(B)))) {
  std::swap(FI, TI);
  Pred = ICmpInst::getSwappedPredicate(Pred);
}

// With any pair of no-wrap subtracts:
// (A > B) ? (A - B) : (B - A) --> abs(A - B)
if (Pred == CmpInst::ICMP_SGT &&
    match(TI, m_Sub(m_Specific(A), m_Specific(B))) &&
    match(FI, m_Sub(m_Specific(B), m_Specific(A))) &&
    (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap()) &&
    (FI->hasNoSignedWrap() || FI->hasNoUnsignedWrap())) {
  // The remaining subtract is not "nuw" any more.
  // If there's one use of the subtract (no other use than the use we are
  // about to replace), then we know that the sub is "nsw" in this context
  // even if it was only "nuw" before. If there's another use, then we can't
  // add "nsw" to the existing instruction because it may not be safe in the
  // other user's context.
  TI->setHasNoUnsignedWrap(false);
  if (!TI->hasNoSignedWrap())
    TI->setHasNoSignedWrap(TI->hasOneUse());
  return Builder.CreateBinaryIntrinsic(Intrinsic::abs, TI, Builder.getTrue());
}

return nullptr;
992}

994/// Fold the following code sequence:
995/// \code
996///   int a = ctlz(x & -x);
997//    x ? 31 - a : a;
998/// \code
999///
1000/// into:
1001///   cttz(x)
1002static Instruction *foldSelectCtlzToCttz(ICmpInst *ICI, Value *TrueVal,
                                       Value *FalseVal,
                                       InstCombiner::BuilderTy &Builder) {
unsigned BitWidth = TrueVal->getType()->getScalarSizeInBits();
if (!ICI->isEquality() || !match(ICI->getOperand(1), m_Zero()))
  return nullptr;

if (ICI->getPredicate() == ICmpInst::ICMP_NE)
  std::swap(TrueVal, FalseVal);

if (!match(FalseVal,
           m_Xor(m_Deferred(TrueVal), m_SpecificInt(BitWidth - 1))))
  return nullptr;

if (!match(TrueVal, m_Intrinsic<Intrinsic::ctlz>()))
  return nullptr;

Value *X = ICI->getOperand(0);
auto *II = cast<IntrinsicInst>(TrueVal);
if (!match(II->getOperand(0), m_c_And(m_Specific(X), m_Neg(m_Specific(X)))))
  return nullptr;

Function *F = Intrinsic::getDeclaration(II->getModule(), Intrinsic::cttz,
                                        II->getType());
return CallInst::Create(F, {X, II->getArgOperand(1)});
1027}

1029/// Attempt to fold a cttz/ctlz followed by a icmp plus select into a single
1030/// call to cttz/ctlz with flag 'is_zero_poison' cleared.
1031///
1032/// For example, we can fold the following code sequence:
1033/// \code
1034///   %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 true)
1035///   %1 = icmp ne i32 %x, 0
1036///   %2 = select i1 %1, i32 %0, i32 32
1037/// \code
1038///
1039/// into:
1040///   %0 = tail call i32 @llvm.cttz.i32(i32 %x, i1 false)
1041static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal,
                               InstCombiner::BuilderTy &Builder) {
ICmpInst::Predicate Pred = ICI->getPredicate();
Value *CmpLHS = ICI->getOperand(0);
Value *CmpRHS = ICI->getOperand(1);

// Check if the select condition compares a value for equality.
if (!ICI->isEquality())
  return nullptr;

Value *SelectArg = FalseVal;
Value *ValueOnZero = TrueVal;
if (Pred == ICmpInst::ICMP_NE)
  std::swap(SelectArg, ValueOnZero);

// Skip zero extend/truncate.
Value *Count = nullptr;
if (!match(SelectArg, m_ZExt(m_Value(Count))) &&
    !match(SelectArg, m_Trunc(m_Value(Count))))
  Count = SelectArg;

// Check that 'Count' is a call to intrinsic cttz/ctlz. Also check that the
// input to the cttz/ctlz is used as LHS for the compare instruction.
Value *X;
if (!match(Count, m_Intrinsic<Intrinsic::cttz>(m_Value(X))) &&
    !match(Count, m_Intrinsic<Intrinsic::ctlz>(m_Value(X))))
  return nullptr;

// (X == 0) ? BitWidth : ctz(X)
// (X == -1) ? BitWidth : ctz(~X)
if ((X != CmpLHS || !match(CmpRHS, m_Zero())) &&
    (!match(X, m_Not(m_Specific(CmpLHS))) || !match(CmpRHS, m_AllOnes())))
  return nullptr;

IntrinsicInst *II = cast<IntrinsicInst>(Count);

// Check if the value propagated on zero is a constant number equal to the
// sizeof in bits of 'Count'.
unsigned SizeOfInBits = Count->getType()->getScalarSizeInBits();
if (match(ValueOnZero, m_SpecificInt(SizeOfInBits))) {
  // Explicitly clear the 'is_zero_poison' flag. It's always valid to go from
  // true to false on this flag, so we can replace it for all users.
  II->setArgOperand(1, ConstantInt::getFalse(II->getContext()));
  return SelectArg;
}

// The ValueOnZero is not the bitwidth. But if the cttz/ctlz (and optional
// zext/trunc) have one use (ending at the select), the cttz/ctlz result will
// not be used if the input is zero. Relax to 'zero is poison' for that case.
if (II->hasOneUse() && SelectArg->hasOneUse() &&
    !match(II->getArgOperand(1), m_One()))
  II->setArgOperand(1, ConstantInt::getTrue(II->getContext()));

return nullptr;
1095}

1097/// Return true if we find and adjust an icmp+select pattern where the compare
1098/// is with a constant that can be incremented or decremented to match the
1099/// minimum or maximum idiom.
1100static bool adjustMinMax(SelectInst &Sel, ICmpInst &Cmp) {
ICmpInst::Predicate Pred = Cmp.getPredicate();
Value *CmpLHS = Cmp.getOperand(0);
Value *CmpRHS = Cmp.getOperand(1);
Value *TrueVal = Sel.getTrueValue();
Value *FalseVal = Sel.getFalseValue();

// We may move or edit the compare, so make sure the select is the only user.
const APInt *CmpC;
if (!Cmp.hasOneUse() || !match(CmpRHS, m_APInt(CmpC)))
  return false;

// These transforms only work for selects of integers or vector selects of
// integer vectors.
Type *SelTy = Sel.getType();
auto *SelEltTy = dyn_cast<IntegerType>(SelTy->getScalarType());
if (!SelEltTy || SelTy->isVectorTy() != Cmp.getType()->isVectorTy())
  return false;

Constant *AdjustedRHS;
if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_SGT)
  AdjustedRHS = ConstantInt::get(CmpRHS->getType(), *CmpC + 1);
else if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_SLT)
  AdjustedRHS = ConstantInt::get(CmpRHS->getType(), *CmpC - 1);
else
  return false;

// X > C ? X : C+1  -->  X < C+1 ? C+1 : X
// X < C ? X : C-1  -->  X > C-1 ? C-1 : X
if ((CmpLHS == TrueVal && AdjustedRHS == FalseVal) ||
    (CmpLHS == FalseVal && AdjustedRHS == TrueVal)) {
  ; // Nothing to do here. Values match without any sign/zero extension.
}
// Types do not match. Instead of calculating this with mixed types, promote
// all to the larger type. This enables scalar evolution to analyze this
// expression.
else if (CmpRHS->getType()->getScalarSizeInBits() < SelEltTy->getBitWidth()) {
  Constant *SextRHS = ConstantExpr::getSExt(AdjustedRHS, SelTy);

  // X = sext x; x >s c ? X : C+1 --> X = sext x; X <s C+1 ? C+1 : X
  // X = sext x; x <s c ? X : C-1 --> X = sext x; X >s C-1 ? C-1 : X
  // X = sext x; x >u c ? X : C+1 --> X = sext x; X <u C+1 ? C+1 : X
  // X = sext x; x <u c ? X : C-1 --> X = sext x; X >u C-1 ? C-1 : X
  if (match(TrueVal, m_SExt(m_Specific(CmpLHS))) && SextRHS == FalseVal) {
    CmpLHS = TrueVal;
    AdjustedRHS = SextRHS;
  } else if (match(FalseVal, m_SExt(m_Specific(CmpLHS))) &&
             SextRHS == TrueVal) {
    CmpLHS = FalseVal;
    AdjustedRHS = SextRHS;
  } else if (Cmp.isUnsigned()) {
    Constant *ZextRHS = ConstantExpr::getZExt(AdjustedRHS, SelTy);
    // X = zext x; x >u c ? X : C+1 --> X = zext x; X <u C+1 ? C+1 : X
    // X = zext x; x <u c ? X : C-1 --> X = zext x; X >u C-1 ? C-1 : X
    // zext + signed compare cannot be changed:
    //    0xff <s 0x00, but 0x00ff >s 0x0000
    if (match(TrueVal, m_ZExt(m_Specific(CmpLHS))) && ZextRHS == FalseVal) {
      CmpLHS = TrueVal;
      AdjustedRHS = ZextRHS;
    } else if (match(FalseVal, m_ZExt(m_Specific(CmpLHS))) &&
               ZextRHS == TrueVal) {
      CmpLHS = FalseVal;
      AdjustedRHS = ZextRHS;
    } else {
      return false;
    }
  } else {
    return false;
  }
} else {
  return false;
}

Pred = ICmpInst::getSwappedPredicate(Pred);
CmpRHS = AdjustedRHS;
std::swap(FalseVal, TrueVal);
Cmp.setPredicate(Pred);
Cmp.setOperand(0, CmpLHS);
Cmp.setOperand(1, CmpRHS);
Sel.setOperand(1, TrueVal);
Sel.setOperand(2, FalseVal);
Sel.swapProfMetadata();

// Move the compare instruction right before the select instruction. Otherwise
// the sext/zext value may be defined after the compare instruction uses it.
Cmp.moveBefore(&Sel);

return true;
1188}

1190static Instruction *canonicalizeSPF(SelectInst &Sel, ICmpInst &Cmp,
                                  InstCombinerImpl &IC) {
Value *LHS, *RHS;
// TODO: What to do with pointer min/max patterns?
if (!Sel.getType()->isIntOrIntVectorTy())
  return nullptr;

SelectPatternFlavor SPF = matchSelectPattern(&Sel, LHS, RHS).Flavor;
if (SPF == SelectPatternFlavor::SPF_ABS ||
    SPF == SelectPatternFlavor::SPF_NABS) {
  if (!Cmp.hasOneUse() && !RHS->hasOneUse())
    return nullptr; // TODO: Relax this restriction.

  // Note that NSW flag can only be propagated for normal, non-negated abs!
  bool IntMinIsPoison = SPF == SelectPatternFlavor::SPF_ABS &&
                        match(RHS, m_NSWNeg(m_Specific(LHS)));
  Constant *IntMinIsPoisonC =
      ConstantInt::get(Type::getInt1Ty(Sel.getContext()), IntMinIsPoison);
  Instruction *Abs =
      IC.Builder.CreateBinaryIntrinsic(Intrinsic::abs, LHS, IntMinIsPoisonC);

  if (SPF == SelectPatternFlavor::SPF_NABS)
    return BinaryOperator::CreateNeg(Abs); // Always without NSW flag!
  return IC.replaceInstUsesWith(Sel, Abs);
}

if (SelectPatternResult::isMinOrMax(SPF)) {
  Intrinsic::ID IntrinsicID;
  switch (SPF) {
  case SelectPatternFlavor::SPF_UMIN:
    IntrinsicID = Intrinsic::umin;
    break;
  case SelectPatternFlavor::SPF_UMAX:
    IntrinsicID = Intrinsic::umax;
    break;
  case SelectPatternFlavor::SPF_SMIN:
    IntrinsicID = Intrinsic::smin;
    break;
  case SelectPatternFlavor::SPF_SMAX:
    IntrinsicID = Intrinsic::smax;
    break;
  default:
    llvm_unreachable("Unexpected SPF")::llvm::llvm_unreachable_internal("Unexpected SPF", "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp"
, 1232);
  }
  return IC.replaceInstUsesWith(
      Sel, IC.Builder.CreateBinaryIntrinsic(IntrinsicID, LHS, RHS));
}

return nullptr;
1239}

1241static bool replaceInInstruction(Value *V, Value *Old, Value *New,
                               InstCombiner &IC, unsigned Depth = 0) {
// Conservatively limit replacement to two instructions upwards.
if (Depth == 2)
  return false;

auto *I = dyn_cast<Instruction>(V);
if (!I || !I->hasOneUse() || !isSafeToSpeculativelyExecute(I))
  return false;

bool Changed = false;
for (Use &U : I->operands()) {
  if (U == Old) {
    IC.replaceUse(U, New);
    Changed = true;
  } else {
    Changed |= replaceInInstruction(U, Old, New, IC, Depth + 1);
  }
}
return Changed;
1261}

1263/// If we have a select with an equality comparison, then we know the value in
1264/// one of the arms of the select. See if substituting this value into an arm
1265/// and simplifying the result yields the same value as the other arm.
1266///
1267/// To make this transform safe, we must drop poison-generating flags
1268/// (nsw, etc) if we simplified to a binop because the select may be guarding
1269/// that poison from propagating. If the existing binop already had no
1270/// poison-generating flags, then this transform can be done by instsimplify.
1271///
1272/// Consider:
1273///   %cmp = icmp eq i32 %x, 2147483647
1274///   %add = add nsw i32 %x, 1
1275///   %sel = select i1 %cmp, i32 -2147483648, i32 %add
1276///
1277/// We can't replace %sel with %add unless we strip away the flags.
1278/// TODO: Wrapping flags could be preserved in some cases with better analysis.
1279Instruction *InstCombinerImpl::foldSelectValueEquivalence(SelectInst &Sel,
                                                        ICmpInst &Cmp) {
if (!Cmp.isEquality())
  return nullptr;

// Canonicalize the pattern to ICMP_EQ by swapping the select operands.
Value *TrueVal = Sel.getTrueValue(), *FalseVal = Sel.getFalseValue();
bool Swapped = false;
if (Cmp.getPredicate() == ICmpInst::ICMP_NE) {
  std::swap(TrueVal, FalseVal);
  Swapped = true;
}

// In X == Y ? f(X) : Z, try to evaluate f(Y) and replace the operand.
// Make sure Y cannot be undef though, as we might pick different values for
// undef in the icmp and in f(Y). Additionally, take care to avoid replacing
// X == Y ? X : Z with X == Y ? Y : Z, as that would lead to an infinite
// replacement cycle.
Value *CmpLHS = Cmp.getOperand(0), *CmpRHS = Cmp.getOperand(1);
if (TrueVal != CmpLHS &&
    isGuaranteedNotToBeUndefOrPoison(CmpRHS, SQ.AC, &Sel, &DT)) {
  if (Value *V = simplifyWithOpReplaced(TrueVal, CmpLHS, CmpRHS, SQ,
                                        /* AllowRefinement */ true))
    return replaceOperand(Sel, Swapped ? 2 : 1, V);

  // Even if TrueVal does not simplify, we can directly replace a use of
  // CmpLHS with CmpRHS, as long as the instruction is not used anywhere
  // else and is safe to speculatively execute (we may end up executing it
  // with different operands, which should not cause side-effects or trigger
  // undefined behavior). Only do this if CmpRHS is a constant, as
  // profitability is not clear for other cases.
  // FIXME: Support vectors.
  if (match(CmpRHS, m_ImmConstant()) && !match(CmpLHS, m_ImmConstant()) &&
      !Cmp.getType()->isVectorTy())
    if (replaceInInstruction(TrueVal, CmpLHS, CmpRHS, *this))
      return &Sel;
}
if (TrueVal != CmpRHS &&
    isGuaranteedNotToBeUndefOrPoison(CmpLHS, SQ.AC, &Sel, &DT))
  if (Value *V = simplifyWithOpReplaced(TrueVal, CmpRHS, CmpLHS, SQ,
                                        /* AllowRefinement */ true))
    return replaceOperand(Sel, Swapped ? 2 : 1, V);

auto *FalseInst = dyn_cast<Instruction>(FalseVal);
if (!FalseInst)
  return nullptr;

// InstSimplify already performed this fold if it was possible subject to
// current poison-generating flags. Try the transform again with
// poison-generating flags temporarily dropped.
bool WasNUW = false, WasNSW = false, WasExact = false, WasInBounds = false;
if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(FalseVal)) {
  WasNUW = OBO->hasNoUnsignedWrap();
  WasNSW = OBO->hasNoSignedWrap();
  FalseInst->setHasNoUnsignedWrap(false);
  FalseInst->setHasNoSignedWrap(false);
}
if (auto *PEO = dyn_cast<PossiblyExactOperator>(FalseVal)) {
  WasExact = PEO->isExact();
  FalseInst->setIsExact(false);
}
if (auto *GEP = dyn_cast<GetElementPtrInst>(FalseVal)) {
  WasInBounds = GEP->isInBounds();
  GEP->setIsInBounds(false);
}

// Try each equivalence substitution possibility.
// We have an 'EQ' comparison, so the select's false value will propagate.
// Example:
// (X == 42) ? 43 : (X + 1) --> (X == 42) ? (X + 1) : (X + 1) --> X + 1
if (simplifyWithOpReplaced(FalseVal, CmpLHS, CmpRHS, SQ,
                           /* AllowRefinement */ false) == TrueVal ||
    simplifyWithOpReplaced(FalseVal, CmpRHS, CmpLHS, SQ,
                           /* AllowRefinement */ false) == TrueVal) {
  return replaceInstUsesWith(Sel, FalseVal);
}

// Restore poison-generating flags if the transform did not apply.
if (WasNUW)
  FalseInst->setHasNoUnsignedWrap();
if (WasNSW)
  FalseInst->setHasNoSignedWrap();
if (WasExact)
  FalseInst->setIsExact();
if (WasInBounds)
  cast<GetElementPtrInst>(FalseInst)->setIsInBounds();

return nullptr;
1367}

1369// See if this is a pattern like:
1370//   %old_cmp1 = icmp slt i32 %x, C2
1371//   %old_replacement = select i1 %old_cmp1, i32 %target_low, i32 %target_high
1372//   %old_x_offseted = add i32 %x, C1
1373//   %old_cmp0 = icmp ult i32 %old_x_offseted, C0
1374//   %r = select i1 %old_cmp0, i32 %x, i32 %old_replacement
1375// This can be rewritten as more canonical pattern:
1376//   %new_cmp1 = icmp slt i32 %x, -C1
1377//   %new_cmp2 = icmp sge i32 %x, C0-C1
1378//   %new_clamped_low = select i1 %new_cmp1, i32 %target_low, i32 %x
1379//   %r = select i1 %new_cmp2, i32 %target_high, i32 %new_clamped_low
1380// Iff -C1 s<= C2 s<= C0-C1
1381// Also ULT predicate can also be UGT iff C0 != -1 (+invert result)
1382//      SLT predicate can also be SGT iff C2 != INT_MAX (+invert res.)
1383static Value *canonicalizeClampLike(SelectInst &Sel0, ICmpInst &Cmp0,
                                  InstCombiner::BuilderTy &Builder) {
Value *X = Sel0.getTrueValue();
Value *Sel1 = Sel0.getFalseValue();

// First match the condition of the outermost select.
// Said condition must be one-use.
if (!Cmp0.hasOneUse())
  return nullptr;
ICmpInst::Predicate Pred0 = Cmp0.getPredicate();
Value *Cmp00 = Cmp0.getOperand(0);
Constant *C0;
if (!match(Cmp0.getOperand(1),
           m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C0))))
  return nullptr;

if (!isa<SelectInst>(Sel1)) {
  Pred0 = ICmpInst::getInversePredicate(Pred0);
  std::swap(X, Sel1);
}

// Canonicalize Cmp0 into ult or uge.
// FIXME: we shouldn't care about lanes that are 'undef' in the end?
switch (Pred0) {
case ICmpInst::Predicate::ICMP_ULT:
case ICmpInst::Predicate::ICMP_UGE:
  // Although icmp ult %x, 0 is an unusual thing to try and should generally
  // have been simplified, it does not verify with undef inputs so ensure we
  // are not in a strange state.
  if (!match(C0, m_SpecificInt_ICMP(
                     ICmpInst::Predicate::ICMP_NE,
                     APInt::getZero(C0->getType()->getScalarSizeInBits()))))
    return nullptr;
  break; // Great!
case ICmpInst::Predicate::ICMP_ULE:
case ICmpInst::Predicate::ICMP_UGT:
  // We want to canonicalize it to 'ult' or 'uge', so we'll need to increment
  // C0, which again means it must not have any all-ones elements.
  if (!match(C0,
             m_SpecificInt_ICMP(
                 ICmpInst::Predicate::ICMP_NE,
                 APInt::getAllOnes(C0->getType()->getScalarSizeInBits()))))
    return nullptr; // Can't do, have all-ones element[s].
  Pred0 = ICmpInst::getFlippedStrictnessPredicate(Pred0);
  C0 = InstCombiner::AddOne(C0);
  break;
default:
  return nullptr; // Unknown predicate.
}

// Now that we've canonicalized the ICmp, we know the X we expect;
// the select in other hand should be one-use.
if (!Sel1->hasOneUse())
  return nullptr;

// If the types do not match, look through any truncs to the underlying
// instruction.
if (Cmp00->getType() != X->getType() && X->hasOneUse())
  match(X, m_TruncOrSelf(m_Value(X)));

// We now can finish matching the condition of the outermost select:
// it should either be the X itself, or an addition of some constant to X.
Constant *C1;
if (Cmp00 == X)
  C1 = ConstantInt::getNullValue(X->getType());
else if (!match(Cmp00,
                m_Add(m_Specific(X),
                      m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C1)))))
  return nullptr;

Value *Cmp1;
ICmpInst::Predicate Pred1;
Constant *C2;
Value *ReplacementLow, *ReplacementHigh;
if (!match(Sel1, m_Select(m_Value(Cmp1), m_Value(ReplacementLow),
                          m_Value(ReplacementHigh))) ||
    !match(Cmp1,
           m_ICmp(Pred1, m_Specific(X),
                  m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C2)))))
  return nullptr;

if (!Cmp1->hasOneUse() && (Cmp00 == X || !Cmp00->hasOneUse()))
  return nullptr; // Not enough one-use instructions for the fold.
// FIXME: this restriction could be relaxed if Cmp1 can be reused as one of
//        two comparisons we'll need to build.

// Canonicalize Cmp1 into the form we expect.
// FIXME: we shouldn't care about lanes that are 'undef' in the end?
switch (Pred1) {
case ICmpInst::Predicate::ICMP_SLT:
  break;
case ICmpInst::Predicate::ICMP_SLE:
  // We'd have to increment C2 by one, and for that it must not have signed
  // max element, but then it would have been canonicalized to 'slt' before
  // we get here. So we can't do anything useful with 'sle'.
  return nullptr;
case ICmpInst::Predicate::ICMP_SGT:
  // We want to canonicalize it to 'slt', so we'll need to increment C2,
  // which again means it must not have any signed max elements.
  if (!match(C2,
             m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_NE,
                                APInt::getSignedMaxValue(
                                    C2->getType()->getScalarSizeInBits()))))
    return nullptr; // Can't do, have signed max element[s].
  C2 = InstCombiner::AddOne(C2);
  [[fallthrough]];
case ICmpInst::Predicate::ICMP_SGE:
  // Also non-canonical, but here we don't need to change C2,
  // so we don't have any restrictions on C2, so we can just handle it.
  Pred1 = ICmpInst::Predicate::ICMP_SLT;
  std::swap(ReplacementLow, ReplacementHigh);
  break;
default:
  return nullptr; // Unknown predicate.
}
assert(Pred1 == ICmpInst::Predicate::ICMP_SLT &&(static_cast <bool> (Pred1 == ICmpInst::Predicate::ICMP_SLT
 && "Unexpected predicate type.") ? void (0) : __assert_fail
 ("Pred1 == ICmpInst::Predicate::ICMP_SLT && \"Unexpected predicate type.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 1499
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected predicate type.")(static_cast <bool> (Pred1 == ICmpInst::Predicate::ICMP_SLT
 && "Unexpected predicate type.") ? void (0) : __assert_fail
 ("Pred1 == ICmpInst::Predicate::ICMP_SLT && \"Unexpected predicate type.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 1499
, __extension__ __PRETTY_FUNCTION__));

// The thresholds of this clamp-like pattern.
auto *ThresholdLowIncl = ConstantExpr::getNeg(C1);
auto *ThresholdHighExcl = ConstantExpr::getSub(C0, C1);

assert((Pred0 == ICmpInst::Predicate::ICMP_ULT ||(static_cast <bool> ((Pred0 == ICmpInst::Predicate::ICMP_ULT
 || Pred0 == ICmpInst::Predicate::ICMP_UGE) && "Unexpected predicate type."
) ? void (0) : __assert_fail ("(Pred0 == ICmpInst::Predicate::ICMP_ULT || Pred0 == ICmpInst::Predicate::ICMP_UGE) && \"Unexpected predicate type.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 1507
, __extension__ __PRETTY_FUNCTION__))
        Pred0 == ICmpInst::Predicate::ICMP_UGE) &&(static_cast <bool> ((Pred0 == ICmpInst::Predicate::ICMP_ULT
 || Pred0 == ICmpInst::Predicate::ICMP_UGE) && "Unexpected predicate type."
) ? void (0) : __assert_fail ("(Pred0 == ICmpInst::Predicate::ICMP_ULT || Pred0 == ICmpInst::Predicate::ICMP_UGE) && \"Unexpected predicate type.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 1507
, __extension__ __PRETTY_FUNCTION__))
       "Unexpected predicate type.")(static_cast <bool> ((Pred0 == ICmpInst::Predicate::ICMP_ULT
 || Pred0 == ICmpInst::Predicate::ICMP_UGE) && "Unexpected predicate type."
) ? void (0) : __assert_fail ("(Pred0 == ICmpInst::Predicate::ICMP_ULT || Pred0 == ICmpInst::Predicate::ICMP_UGE) && \"Unexpected predicate type.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 1507
, __extension__ __PRETTY_FUNCTION__));
if (Pred0 == ICmpInst::Predicate::ICMP_UGE)
  std::swap(ThresholdLowIncl, ThresholdHighExcl);

// The fold has a precondition 1: C2 s>= ThresholdLow
auto *Precond1 = ConstantExpr::getICmp(ICmpInst::Predicate::ICMP_SGE, C2,
                                       ThresholdLowIncl);
if (!match(Precond1, m_One()))
  return nullptr;
// The fold has a precondition 2: C2 s<= ThresholdHigh
auto *Precond2 = ConstantExpr::getICmp(ICmpInst::Predicate::ICMP_SLE, C2,
                                       ThresholdHighExcl);
if (!match(Precond2, m_One()))
  return nullptr;

// If we are matching from a truncated input, we need to sext the
// ReplacementLow and ReplacementHigh values. Only do the transform if they
// are free to extend due to being constants.
if (X->getType() != Sel0.getType()) {
  Constant *LowC, *HighC;
  if (!match(ReplacementLow, m_ImmConstant(LowC)) ||
      !match(ReplacementHigh, m_ImmConstant(HighC)))
    return nullptr;
  ReplacementLow = ConstantExpr::getSExt(LowC, X->getType());
  ReplacementHigh = ConstantExpr::getSExt(HighC, X->getType());
}

// All good, finally emit the new pattern.
Value *ShouldReplaceLow = Builder.CreateICmpSLT(X, ThresholdLowIncl);
Value *ShouldReplaceHigh = Builder.CreateICmpSGE(X, ThresholdHighExcl);
Value *MaybeReplacedLow =
    Builder.CreateSelect(ShouldReplaceLow, ReplacementLow, X);

// Create the final select. If we looked through a truncate above, we will
// need to retruncate the result.
Value *MaybeReplacedHigh = Builder.CreateSelect(
    ShouldReplaceHigh, ReplacementHigh, MaybeReplacedLow);
return Builder.CreateTrunc(MaybeReplacedHigh, Sel0.getType());
1545}

1547// If we have
1548//  %cmp = icmp [canonical predicate] i32 %x, C0
1549//  %r = select i1 %cmp, i32 %y, i32 C1
1550// Where C0 != C1 and %x may be different from %y, see if the constant that we
1551// will have if we flip the strictness of the predicate (i.e. without changing
1552// the result) is identical to the C1 in select. If it matches we can change
1553// original comparison to one with swapped predicate, reuse the constant,
1554// and swap the hands of select.
1555static Instruction *
1556tryToReuseConstantFromSelectInComparison(SelectInst &Sel, ICmpInst &Cmp,
                                       InstCombinerImpl &IC) {
ICmpInst::Predicate Pred;
Value *X;
Constant *C0;
if (!match(&Cmp, m_OneUse(m_ICmp(
                     Pred, m_Value(X),
                     m_CombineAnd(m_AnyIntegralConstant(), m_Constant(C0))))))
  return nullptr;

// If comparison predicate is non-relational, we won't be able to do anything.
if (ICmpInst::isEquality(Pred))
  return nullptr;

// If comparison predicate is non-canonical, then we certainly won't be able
// to make it canonical; canonicalizeCmpWithConstant() already tried.
if (!InstCombiner::isCanonicalPredicate(Pred))
  return nullptr;

// If the [input] type of comparison and select type are different, lets abort
// for now. We could try to compare constants with trunc/[zs]ext though.
if (C0->getType() != Sel.getType())
  return nullptr;

// ULT with 'add' of a constant is canonical. See foldICmpAddConstant().
// FIXME: Are there more magic icmp predicate+constant pairs we must avoid?
//        Or should we just abandon this transform entirely?
if (Pred == CmpInst::ICMP_ULT && match(X, m_Add(m_Value(), m_Constant())))
  return nullptr;


Value *SelVal0, *SelVal1; // We do not care which one is from where.
match(&Sel, m_Select(m_Value(), m_Value(SelVal0), m_Value(SelVal1)));
// At least one of these values we are selecting between must be a constant
// else we'll never succeed.
if (!match(SelVal0, m_AnyIntegralConstant()) &&
    !match(SelVal1, m_AnyIntegralConstant()))
  return nullptr;

// Does this constant C match any of the `select` values?
auto MatchesSelectValue = [SelVal0, SelVal1](Constant *C) {
  return C->isElementWiseEqual(SelVal0) || C->isElementWiseEqual(SelVal1);
};

// If C0 *already* matches true/false value of select, we are done.
if (MatchesSelectValue(C0))
  return nullptr;

// Check the constant we'd have with flipped-strictness predicate.
auto FlippedStrictness =
    InstCombiner::getFlippedStrictnessPredicateAndConstant(Pred, C0);
if (!FlippedStrictness)
  return nullptr;

// If said constant doesn't match either, then there is no hope,
if (!MatchesSelectValue(FlippedStrictness->second))
  return nullptr;

// It matched! Lets insert the new comparison just before select.
InstCombiner::BuilderTy::InsertPointGuard Guard(IC.Builder);
IC.Builder.SetInsertPoint(&Sel);

Pred = ICmpInst::getSwappedPredicate(Pred); // Yes, swapped.
Value *NewCmp = IC.Builder.CreateICmp(Pred, X, FlippedStrictness->second,
                                      Cmp.getName() + ".inv");
IC.replaceOperand(Sel, 0, NewCmp);
Sel.swapValues();
Sel.swapProfMetadata();

return &Sel;
1626}

1628static Instruction *foldSelectZeroOrOnes(ICmpInst *Cmp, Value *TVal,
                                       Value *FVal,
                                       InstCombiner::BuilderTy &Builder) {
if (!Cmp->hasOneUse())
  return nullptr;

const APInt *CmpC;
if (!match(Cmp->getOperand(1), m_APIntAllowUndef(CmpC)))
  return nullptr;

// (X u< 2) ? -X : -1 --> sext (X != 0)
Value *X = Cmp->getOperand(0);
if (Cmp->getPredicate() == ICmpInst::ICMP_ULT && *CmpC == 2 &&
    match(TVal, m_Neg(m_Specific(X))) && match(FVal, m_AllOnes()))
  return new SExtInst(Builder.CreateIsNotNull(X), TVal->getType());

// (X u> 1) ? -1 : -X --> sext (X != 0)
if (Cmp->getPredicate() == ICmpInst::ICMP_UGT && *CmpC == 1 &&
    match(FVal, m_Neg(m_Specific(X))) && match(TVal, m_AllOnes()))
  return new SExtInst(Builder.CreateIsNotNull(X), TVal->getType());

return nullptr;
1650}

1652static Value *foldSelectInstWithICmpConst(SelectInst &SI, ICmpInst *ICI,
                                        InstCombiner::BuilderTy &Builder) {
const APInt *CmpC;
Value *V;
CmpInst::Predicate Pred;
if (!match(ICI, m_ICmp(Pred, m_Value(V), m_APInt(CmpC))))
  return nullptr;

// Match clamp away from min/max value as a max/min operation.
Value *TVal = SI.getTrueValue();
Value *FVal = SI.getFalseValue();
if (Pred == ICmpInst::ICMP_EQ && V == FVal) {
  // (V == UMIN) ? UMIN+1 : V --> umax(V, UMIN+1)
  if (CmpC->isMinValue() && match(TVal, m_SpecificInt(*CmpC + 1)))
    return Builder.CreateBinaryIntrinsic(Intrinsic::umax, V, TVal);
  // (V == UMAX) ? UMAX-1 : V --> umin(V, UMAX-1)
  if (CmpC->isMaxValue() && match(TVal, m_SpecificInt(*CmpC - 1)))
    return Builder.CreateBinaryIntrinsic(Intrinsic::umin, V, TVal);
  // (V == SMIN) ? SMIN+1 : V --> smax(V, SMIN+1)
  if (CmpC->isMinSignedValue() && match(TVal, m_SpecificInt(*CmpC + 1)))
    return Builder.CreateBinaryIntrinsic(Intrinsic::smax, V, TVal);
  // (V == SMAX) ? SMAX-1 : V --> smin(V, SMAX-1)
  if (CmpC->isMaxSignedValue() && match(TVal, m_SpecificInt(*CmpC - 1)))
    return Builder.CreateBinaryIntrinsic(Intrinsic::smin, V, TVal);
}

BinaryOperator *BO;
const APInt *C;
CmpInst::Predicate CPred;
if (match(&SI, m_Select(m_Specific(ICI), m_APInt(C), m_BinOp(BO))))
  CPred = ICI->getPredicate();
else if (match(&SI, m_Select(m_Specific(ICI), m_BinOp(BO), m_APInt(C))))
  CPred = ICI->getInversePredicate();
else
  return nullptr;

const APInt *BinOpC;
if (!match(BO, m_BinOp(m_Specific(V), m_APInt(BinOpC))))
  return nullptr;

ConstantRange R = ConstantRange::makeExactICmpRegion(CPred, *CmpC)
                      .binaryOp(BO->getOpcode(), *BinOpC);
if (R == *C) {
  BO->dropPoisonGeneratingFlags();
  return BO;
}
return nullptr;
1699}

1701/// Visit a SelectInst that has an ICmpInst as its first operand.
1702Instruction *InstCombinerImpl::foldSelectInstWithICmp(SelectInst &SI,
                                                    ICmpInst *ICI) {
if (Instruction *NewSel = foldSelectValueEquivalence(SI, *ICI))
  return NewSel;

if (Instruction *NewSPF = canonicalizeSPF(SI, *ICI, *this))
  return NewSPF;

if (Value *V = foldSelectInstWithICmpConst(SI, ICI, Builder))
  return replaceInstUsesWith(SI, V);

if (Value *V = canonicalizeClampLike(SI, *ICI, Builder))
  return replaceInstUsesWith(SI, V);

if (Instruction *NewSel =
        tryToReuseConstantFromSelectInComparison(SI, *ICI, *this))
  return NewSel;

bool Changed = adjustMinMax(SI, *ICI);

if (Value *V = foldSelectICmpAnd(SI, ICI, Builder))
  return replaceInstUsesWith(SI, V);

// NOTE: if we wanted to, this is where to detect integer MIN/MAX
Value *TrueVal = SI.getTrueValue();
Value *FalseVal = SI.getFalseValue();
ICmpInst::Predicate Pred = ICI->getPredicate();
Value *CmpLHS = ICI->getOperand(0);
Value *CmpRHS = ICI->getOperand(1);
if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS) && !isa<Constant>(CmpLHS)) {
  if (CmpLHS == TrueVal && Pred == ICmpInst::ICMP_EQ) {
    // Transform (X == C) ? X : Y -> (X == C) ? C : Y
    SI.setOperand(1, CmpRHS);
    Changed = true;
  } else if (CmpLHS == FalseVal && Pred == ICmpInst::ICMP_NE) {
    // Transform (X != C) ? Y : X -> (X != C) ? Y : C
    SI.setOperand(2, CmpRHS);
    Changed = true;
  }
}

// Canonicalize a signbit condition to use zero constant by swapping:
// (CmpLHS > -1) ? TV : FV --> (CmpLHS < 0) ? FV : TV
// To avoid conflicts (infinite loops) with other canonicalizations, this is
// not applied with any constant select arm.
if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes()) &&
    !match(TrueVal, m_Constant()) && !match(FalseVal, m_Constant()) &&
    ICI->hasOneUse()) {
  InstCombiner::BuilderTy::InsertPointGuard Guard(Builder);
  Builder.SetInsertPoint(&SI);
  Value *IsNeg = Builder.CreateIsNeg(CmpLHS, ICI->getName());
  replaceOperand(SI, 0, IsNeg);
  SI.swapValues();
  SI.swapProfMetadata();
  return &SI;
}

// FIXME: This code is nearly duplicated in InstSimplify. Using/refactoring
// decomposeBitTestICmp() might help.
if (TrueVal->getType()->isIntOrIntVectorTy()) {
  unsigned BitWidth =
      DL.getTypeSizeInBits(TrueVal->getType()->getScalarType());
  APInt MinSignedValue = APInt::getSignedMinValue(BitWidth);
  Value *X;
  const APInt *Y, *C;
  bool TrueWhenUnset;
  bool IsBitTest = false;
  if (ICmpInst::isEquality(Pred) &&
      match(CmpLHS, m_And(m_Value(X), m_Power2(Y))) &&
      match(CmpRHS, m_Zero())) {
    IsBitTest = true;
    TrueWhenUnset = Pred == ICmpInst::ICMP_EQ;
  } else if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, m_Zero())) {
    X = CmpLHS;
    Y = &MinSignedValue;
    IsBitTest = true;
    TrueWhenUnset = false;
  } else if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, m_AllOnes())) {
    X = CmpLHS;
    Y = &MinSignedValue;
    IsBitTest = true;
    TrueWhenUnset = true;
  }
  if (IsBitTest) {
    Value *V = nullptr;
    // (X & Y) == 0 ? X : X ^ Y  --> X & ~Y
    if (TrueWhenUnset && TrueVal == X &&
        match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
      V = Builder.CreateAnd(X, ~(*Y));
    // (X & Y) != 0 ? X ^ Y : X  --> X & ~Y
    else if (!TrueWhenUnset && FalseVal == X &&
             match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
      V = Builder.CreateAnd(X, ~(*Y));
    // (X & Y) == 0 ? X ^ Y : X  --> X | Y
    else if (TrueWhenUnset && FalseVal == X &&
             match(TrueVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
      V = Builder.CreateOr(X, *Y);
    // (X & Y) != 0 ? X : X ^ Y  --> X | Y
    else if (!TrueWhenUnset && TrueVal == X &&
             match(FalseVal, m_Xor(m_Specific(X), m_APInt(C))) && *Y == *C)
      V = Builder.CreateOr(X, *Y);

    if (V)
      return replaceInstUsesWith(SI, V);
  }
}

if (Instruction *V =
        foldSelectICmpAndAnd(SI.getType(), ICI, TrueVal, FalseVal, Builder))
  return V;

if (Instruction *V = foldSelectCtlzToCttz(ICI, TrueVal, FalseVal, Builder))
  return V;

if (Instruction *V = foldSelectZeroOrOnes(ICI, TrueVal, FalseVal, Builder))
  return V;

if (Value *V = foldSelectICmpAndOr(ICI, TrueVal, FalseVal, Builder))
  return replaceInstUsesWith(SI, V);

if (Value *V = foldSelectICmpLshrAshr(ICI, TrueVal, FalseVal, Builder))
  return replaceInstUsesWith(SI, V);

if (Value *V = foldSelectCttzCtlz(ICI, TrueVal, FalseVal, Builder))
  return replaceInstUsesWith(SI, V);

if (Value *V = canonicalizeSaturatedSubtract(ICI, TrueVal, FalseVal, Builder))
  return replaceInstUsesWith(SI, V);

if (Value *V = canonicalizeSaturatedAdd(ICI, TrueVal, FalseVal, Builder))
  return replaceInstUsesWith(SI, V);

if (Value *V = foldAbsDiff(ICI, TrueVal, FalseVal, Builder))
  return replaceInstUsesWith(SI, V);

return Changed ? &SI : nullptr;
1838}

1840/// SI is a select whose condition is a PHI node (but the two may be in
1841/// different blocks). See if the true/false values (V) are live in all of the
1842/// predecessor blocks of the PHI. For example, cases like this can't be mapped:
1843///
1844///   X = phi [ C1, BB1], [C2, BB2]
1845///   Y = add
1846///   Z = select X, Y, 0
1847///
1848/// because Y is not live in BB1/BB2.
1849static bool canSelectOperandBeMappingIntoPredBlock(const Value *V,
                                                 const SelectInst &SI) {
// If the value is a non-instruction value like a constant or argument, it
// can always be mapped.
const Instruction *I = dyn_cast<Instruction>(V);
if (!I) return true;

// If V is a PHI node defined in the same block as the condition PHI, we can
// map the arguments.
const PHINode *CondPHI = cast<PHINode>(SI.getCondition());

if (const PHINode *VP = dyn_cast<PHINode>(I))
  if (VP->getParent() == CondPHI->getParent())
    return true;

// Otherwise, if the PHI and select are defined in the same block and if V is
// defined in a different block, then we can transform it.
if (SI.getParent() == CondPHI->getParent() &&
    I->getParent() != CondPHI->getParent())
  return true;

// Otherwise we have a 'hard' case and we can't tell without doing more
// detailed dominator based analysis, punt.
return false;
1873}

1875/// We have an SPF (e.g. a min or max) of an SPF of the form:
1876///   SPF2(SPF1(A, B), C)
1877Instruction *InstCombinerImpl::foldSPFofSPF(Instruction *Inner,
                                          SelectPatternFlavor SPF1, Value *A,
                                          Value *B, Instruction &Outer,
                                          SelectPatternFlavor SPF2,
                                          Value *C) {
if (Outer.getType() != Inner->getType())
  return nullptr;

if (C == A || C == B) {
  // MAX(MAX(A, B), B) -> MAX(A, B)
  // MIN(MIN(a, b), a) -> MIN(a, b)
  // TODO: This could be done in instsimplify.
  if (SPF1 == SPF2 && SelectPatternResult::isMinOrMax(SPF1))
    return replaceInstUsesWith(Outer, Inner);
}

return nullptr;
1894}

1896/// Turn select C, (X + Y), (X - Y) --> (X + (select C, Y, (-Y))).
1897/// This is even legal for FP.
1898static Instruction *foldAddSubSelect(SelectInst &SI,
                                   InstCombiner::BuilderTy &Builder) {
Value *CondVal = SI.getCondition();
Value *TrueVal = SI.getTrueValue();
Value *FalseVal = SI.getFalseValue();
auto *TI = dyn_cast<Instruction>(TrueVal);
auto *FI = dyn_cast<Instruction>(FalseVal);
if (!TI || !FI || !TI->hasOneUse() || !FI->hasOneUse())
  return nullptr;

Instruction *AddOp = nullptr, *SubOp = nullptr;
if ((TI->getOpcode() == Instruction::Sub &&
     FI->getOpcode() == Instruction::Add) ||
    (TI->getOpcode() == Instruction::FSub &&
     FI->getOpcode() == Instruction::FAdd)) {
  AddOp = FI;
  SubOp = TI;
} else if ((FI->getOpcode() == Instruction::Sub &&
            TI->getOpcode() == Instruction::Add) ||
           (FI->getOpcode() == Instruction::FSub &&
            TI->getOpcode() == Instruction::FAdd)) {
  AddOp = TI;
  SubOp = FI;
}

if (AddOp) {
  Value *OtherAddOp = nullptr;
  if (SubOp->getOperand(0) == AddOp->getOperand(0)) {
    OtherAddOp = AddOp->getOperand(1);
  } else if (SubOp->getOperand(0) == AddOp->getOperand(1)) {
    OtherAddOp = AddOp->getOperand(0);
  }

  if (OtherAddOp) {
    // So at this point we know we have (Y -> OtherAddOp):
    //        select C, (add X, Y), (sub X, Z)
    Value *NegVal; // Compute -Z
    if (SI.getType()->isFPOrFPVectorTy()) {
      NegVal = Builder.CreateFNeg(SubOp->getOperand(1));
      if (Instruction *NegInst = dyn_cast<Instruction>(NegVal)) {
        FastMathFlags Flags = AddOp->getFastMathFlags();
        Flags &= SubOp->getFastMathFlags();
        NegInst->setFastMathFlags(Flags);
      }
    } else {
      NegVal = Builder.CreateNeg(SubOp->getOperand(1));
    }

    Value *NewTrueOp = OtherAddOp;
    Value *NewFalseOp = NegVal;
    if (AddOp != TI)
      std::swap(NewTrueOp, NewFalseOp);
    Value *NewSel = Builder.CreateSelect(CondVal, NewTrueOp, NewFalseOp,
                                         SI.getName() + ".p", &SI);

    if (SI.getType()->isFPOrFPVectorTy()) {
      Instruction *RI =
          BinaryOperator::CreateFAdd(SubOp->getOperand(0), NewSel);

      FastMathFlags Flags = AddOp->getFastMathFlags();
      Flags &= SubOp->getFastMathFlags();
      RI->setFastMathFlags(Flags);
      return RI;
    } else
      return BinaryOperator::CreateAdd(SubOp->getOperand(0), NewSel);
  }
}
return nullptr;
1966}

1968/// Turn X + Y overflows ? -1 : X + Y -> uadd_sat X, Y
1969/// And X - Y overflows ? 0 : X - Y -> usub_sat X, Y
1970/// Along with a number of patterns similar to:
1971/// X + Y overflows ? (X < 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
1972/// X - Y overflows ? (X > 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
1973static Instruction *
1974foldOverflowingAddSubSelect(SelectInst &SI, InstCombiner::BuilderTy &Builder) {
Value *CondVal = SI.getCondition();
Value *TrueVal = SI.getTrueValue();
Value *FalseVal = SI.getFalseValue();

WithOverflowInst *II;
if (!match(CondVal, m_ExtractValue<1>(m_WithOverflowInst(II))) ||
    !match(FalseVal, m_ExtractValue<0>(m_Specific(II))))
  return nullptr;

Value *X = II->getLHS();
Value *Y = II->getRHS();

auto IsSignedSaturateLimit = [&](Value *Limit, bool IsAdd) {
  Type *Ty = Limit->getType();

  ICmpInst::Predicate Pred;
  Value *TrueVal, *FalseVal, *Op;
  const APInt *C;
  if (!match(Limit, m_Select(m_ICmp(Pred, m_Value(Op), m_APInt(C)),
                             m_Value(TrueVal), m_Value(FalseVal))))
    return false;

  auto IsZeroOrOne = [](const APInt &C) { return C.isZero() || C.isOne(); };
  auto IsMinMax = [&](Value *Min, Value *Max) {
    APInt MinVal = APInt::getSignedMinValue(Ty->getScalarSizeInBits());
    APInt MaxVal = APInt::getSignedMaxValue(Ty->getScalarSizeInBits());
    return match(Min, m_SpecificInt(MinVal)) &&
           match(Max, m_SpecificInt(MaxVal));
  };

  if (Op != X && Op != Y)
    return false;

  if (IsAdd) {
    // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
    // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
    // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
    // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
    if (Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) &&
        IsMinMax(TrueVal, FalseVal))
      return true;
    // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
    // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
    // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
    // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
    if (Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) &&
        IsMinMax(FalseVal, TrueVal))
      return true;
  } else {
    // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
    // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
    if (Op == X && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C + 1) &&
        IsMinMax(TrueVal, FalseVal))
      return true;
    // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
    // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
    if (Op == X && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 2) &&
        IsMinMax(FalseVal, TrueVal))
      return true;
    // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
    // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
    if (Op == Y && Pred == ICmpInst::ICMP_SLT && IsZeroOrOne(*C) &&
        IsMinMax(FalseVal, TrueVal))
      return true;
    // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
    // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
    if (Op == Y && Pred == ICmpInst::ICMP_SGT && IsZeroOrOne(*C + 1) &&
        IsMinMax(TrueVal, FalseVal))
      return true;
  }

  return false;
};

Intrinsic::ID NewIntrinsicID;
if (II->getIntrinsicID() == Intrinsic::uadd_with_overflow &&
    match(TrueVal, m_AllOnes()))
  // X + Y overflows ? -1 : X + Y -> uadd_sat X, Y
  NewIntrinsicID = Intrinsic::uadd_sat;
else if (II->getIntrinsicID() == Intrinsic::usub_with_overflow &&
         match(TrueVal, m_Zero()))
  // X - Y overflows ? 0 : X - Y -> usub_sat X, Y
  NewIntrinsicID = Intrinsic::usub_sat;
else if (II->getIntrinsicID() == Intrinsic::sadd_with_overflow &&
         IsSignedSaturateLimit(TrueVal, /*IsAdd=*/true))
  // X + Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (X <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (X >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (Y <s 0 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (Y <s 1 ? INTMIN : INTMAX) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (Y >s 0 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
  // X + Y overflows ? (Y >s -1 ? INTMAX : INTMIN) : X + Y --> sadd_sat X, Y
  NewIntrinsicID = Intrinsic::sadd_sat;
else if (II->getIntrinsicID() == Intrinsic::ssub_with_overflow &&
         IsSignedSaturateLimit(TrueVal, /*IsAdd=*/false))
  // X - Y overflows ? (X <s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (X <s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (X >s -1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (X >s -2 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (Y <s 0 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (Y <s 1 ? INTMAX : INTMIN) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (Y >s 0 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
  // X - Y overflows ? (Y >s -1 ? INTMIN : INTMAX) : X - Y --> ssub_sat X, Y
  NewIntrinsicID = Intrinsic::ssub_sat;
else
  return nullptr;

Function *F =
    Intrinsic::getDeclaration(SI.getModule(), NewIntrinsicID, SI.getType());
return CallInst::Create(F, {X, Y});
2086}

2088Instruction *InstCombinerImpl::foldSelectExtConst(SelectInst &Sel) {
Constant *C;
if (!match(Sel.getTrueValue(), m_Constant(C)) &&
    !match(Sel.getFalseValue(), m_Constant(C)))
  return nullptr;

Instruction *ExtInst;
if (!match(Sel.getTrueValue(), m_Instruction(ExtInst)) &&
    !match(Sel.getFalseValue(), m_Instruction(ExtInst)))
  return nullptr;

auto ExtOpcode = ExtInst->getOpcode();
if (ExtOpcode != Instruction::ZExt && ExtOpcode != Instruction::SExt)
  return nullptr;

// If we are extending from a boolean type or if we can create a select that
// has the same size operands as its condition, try to narrow the select.
Value *X = ExtInst->getOperand(0);
Type *SmallType = X->getType();
Value *Cond = Sel.getCondition();
auto *Cmp = dyn_cast<CmpInst>(Cond);
if (!SmallType->isIntOrIntVectorTy(1) &&
    (!Cmp || Cmp->getOperand(0)->getType() != SmallType))
  return nullptr;

// If the constant is the same after truncation to the smaller type and
// extension to the original type, we can narrow the select.
Type *SelType = Sel.getType();
Constant *TruncC = ConstantExpr::getTrunc(C, SmallType);
Constant *ExtC = ConstantExpr::getCast(ExtOpcode, TruncC, SelType);
if (ExtC == C && ExtInst->hasOneUse()) {
  Value *TruncCVal = cast<Value>(TruncC);
  if (ExtInst == Sel.getFalseValue())
    std::swap(X, TruncCVal);

  // select Cond, (ext X), C --> ext(select Cond, X, C')
  // select Cond, C, (ext X) --> ext(select Cond, C', X)
  Value *NewSel = Builder.CreateSelect(Cond, X, TruncCVal, "narrow", &Sel);
  return CastInst::Create(Instruction::CastOps(ExtOpcode), NewSel, SelType);
}

// If one arm of the select is the extend of the condition, replace that arm
// with the extension of the appropriate known bool value.
if (Cond == X) {
  if (ExtInst == Sel.getTrueValue()) {
    // select X, (sext X), C --> select X, -1, C
    // select X, (zext X), C --> select X,  1, C
    Constant *One = ConstantInt::getTrue(SmallType);
    Constant *AllOnesOrOne = ConstantExpr::getCast(ExtOpcode, One, SelType);
    return SelectInst::Create(Cond, AllOnesOrOne, C, "", nullptr, &Sel);
  } else {
    // select X, C, (sext X) --> select X, C, 0
    // select X, C, (zext X) --> select X, C, 0
    Constant *Zero = ConstantInt::getNullValue(SelType);
    return SelectInst::Create(Cond, C, Zero, "", nullptr, &Sel);
  }
}

return nullptr;
2147}

2149/// Try to transform a vector select with a constant condition vector into a
2150/// shuffle for easier combining with other shuffles and insert/extract.
2151static Instruction *canonicalizeSelectToShuffle(SelectInst &SI) {
Value *CondVal = SI.getCondition();
Constant *CondC;
auto *CondValTy = dyn_cast<FixedVectorType>(CondVal->getType());
if (!CondValTy || !match(CondVal, m_Constant(CondC)))
  return nullptr;

unsigned NumElts = CondValTy->getNumElements();
SmallVector<int, 16> Mask;
Mask.reserve(NumElts);
for (unsigned i = 0; i != NumElts; ++i) {
  Constant *Elt = CondC->getAggregateElement(i);
  if (!Elt)
    return nullptr;

  if (Elt->isOneValue()) {
    // If the select condition element is true, choose from the 1st vector.
    Mask.push_back(i);
  } else if (Elt->isNullValue()) {
    // If the select condition element is false, choose from the 2nd vector.
    Mask.push_back(i + NumElts);
  } else if (isa<UndefValue>(Elt)) {
    // Undef in a select condition (choose one of the operands) does not mean
    // the same thing as undef in a shuffle mask (any value is acceptable), so
    // give up.
    return nullptr;
  } else {
    // Bail out on a constant expression.
    return nullptr;
  }
}

return new ShuffleVectorInst(SI.getTrueValue(), SI.getFalseValue(), Mask);
2184}

2186/// If we have a select of vectors with a scalar condition, try to convert that
2187/// to a vector select by splatting the condition. A splat may get folded with
2188/// other operations in IR and having all operands of a select be vector types
2189/// is likely better for vector codegen.
2190static Instruction *canonicalizeScalarSelectOfVecs(SelectInst &Sel,
                                                 InstCombinerImpl &IC) {
auto *Ty = dyn_cast<VectorType>(Sel.getType());
if (!Ty)
  return nullptr;

// We can replace a single-use extract with constant index.
Value *Cond = Sel.getCondition();
if (!match(Cond, m_OneUse(m_ExtractElt(m_Value(), m_ConstantInt()))))
  return nullptr;

// select (extelt V, Index), T, F --> select (splat V, Index), T, F
// Splatting the extracted condition reduces code (we could directly create a
// splat shuffle of the source vector to eliminate the intermediate step).
return IC.replaceOperand(
    Sel, 0, IC.Builder.CreateVectorSplat(Ty->getElementCount(), Cond));
2206}

2208/// Reuse bitcasted operands between a compare and select:
2209/// select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) -->
2210/// bitcast (select (cmp (bitcast C), (bitcast D)), (bitcast C), (bitcast D))
2211static Instruction *foldSelectCmpBitcasts(SelectInst &Sel,
                                        InstCombiner::BuilderTy &Builder) {
Value *Cond = Sel.getCondition();
Value *TVal = Sel.getTrueValue();
Value *FVal = Sel.getFalseValue();

CmpInst::Predicate Pred;
Value *A, *B;
if (!match(Cond, m_Cmp(Pred, m_Value(A), m_Value(B))))
  return nullptr;

// The select condition is a compare instruction. If the select's true/false
// values are already the same as the compare operands, there's nothing to do.
if (TVal == A || TVal == B || FVal == A || FVal == B)
  return nullptr;

Value *C, *D;
if (!match(A, m_BitCast(m_Value(C))) || !match(B, m_BitCast(m_Value(D))))
  return nullptr;

// select (cmp (bitcast C), (bitcast D)), (bitcast TSrc), (bitcast FSrc)
Value *TSrc, *FSrc;
if (!match(TVal, m_BitCast(m_Value(TSrc))) ||
    !match(FVal, m_BitCast(m_Value(FSrc))))
  return nullptr;

// If the select true/false values are *different bitcasts* of the same source
// operands, make the select operands the same as the compare operands and
// cast the result. This is the canonical select form for min/max.
Value *NewSel;
if (TSrc == C && FSrc == D) {
  // select (cmp (bitcast C), (bitcast D)), (bitcast' C), (bitcast' D) -->
  // bitcast (select (cmp A, B), A, B)
  NewSel = Builder.CreateSelect(Cond, A, B, "", &Sel);
} else if (TSrc == D && FSrc == C) {
  // select (cmp (bitcast C), (bitcast D)), (bitcast' D), (bitcast' C) -->
  // bitcast (select (cmp A, B), B, A)
  NewSel = Builder.CreateSelect(Cond, B, A, "", &Sel);
} else {
  return nullptr;
}
return CastInst::CreateBitOrPointerCast(NewSel, Sel.getType());
2253}

2255/// Try to eliminate select instructions that test the returned flag of cmpxchg
2256/// instructions.
2257///
2258/// If a select instruction tests the returned flag of a cmpxchg instruction and
2259/// selects between the returned value of the cmpxchg instruction its compare
2260/// operand, the result of the select will always be equal to its false value.
2261/// For example:
2262///
2263///   %0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
2264///   %1 = extractvalue { i64, i1 } %0, 1
2265///   %2 = extractvalue { i64, i1 } %0, 0
2266///   %3 = select i1 %1, i64 %compare, i64 %2
2267///   ret i64 %3
2268///
2269/// The returned value of the cmpxchg instruction (%2) is the original value
2270/// located at %ptr prior to any update. If the cmpxchg operation succeeds, %2
2271/// must have been equal to %compare. Thus, the result of the select is always
2272/// equal to %2, and the code can be simplified to:
2273///
2274///   %0 = cmpxchg i64* %ptr, i64 %compare, i64 %new_value seq_cst seq_cst
2275///   %1 = extractvalue { i64, i1 } %0, 0
2276///   ret i64 %1
2277///
2278static Value *foldSelectCmpXchg(SelectInst &SI) {
// A helper that determines if V is an extractvalue instruction whose
// aggregate operand is a cmpxchg instruction and whose single index is equal
// to I. If such conditions are true, the helper returns the cmpxchg
// instruction; otherwise, a nullptr is returned.
auto isExtractFromCmpXchg = [](Value *V, unsigned I) -> AtomicCmpXchgInst * {
  auto *Extract = dyn_cast<ExtractValueInst>(V);
  if (!Extract)
    return nullptr;
  if (Extract->getIndices()[0] != I)
    return nullptr;
  return dyn_cast<AtomicCmpXchgInst>(Extract->getAggregateOperand());
};

// If the select has a single user, and this user is a select instruction that
// we can simplify, skip the cmpxchg simplification for now.
if (SI.hasOneUse())
  if (auto *Select = dyn_cast<SelectInst>(SI.user_back()))
    if (Select->getCondition() == SI.getCondition())
      if (Select->getFalseValue() == SI.getTrueValue() ||
          Select->getTrueValue() == SI.getFalseValue())
        return nullptr;

// Ensure the select condition is the returned flag of a cmpxchg instruction.
auto *CmpXchg = isExtractFromCmpXchg(SI.getCondition(), 1);
if (!CmpXchg)
  return nullptr;

// Check the true value case: The true value of the select is the returned
// value of the same cmpxchg used by the condition, and the false value is the
// cmpxchg instruction's compare operand.
if (auto *X = isExtractFromCmpXchg(SI.getTrueValue(), 0))
  if (X == CmpXchg && X->getCompareOperand() == SI.getFalseValue())
    return SI.getFalseValue();

// Check the false value case: The false value of the select is the returned
// value of the same cmpxchg used by the condition, and the true value is the
// cmpxchg instruction's compare operand.
if (auto *X = isExtractFromCmpXchg(SI.getFalseValue(), 0))
  if (X == CmpXchg && X->getCompareOperand() == SI.getTrueValue())
    return SI.getFalseValue();

return nullptr;
2321}

2323/// Try to reduce a funnel/rotate pattern that includes a compare and select
2324/// into a funnel shift intrinsic. Example:
2325/// rotl32(a, b) --> (b == 0 ? a : ((a >> (32 - b)) | (a << b)))
2326///              --> call llvm.fshl.i32(a, a, b)
2327/// fshl32(a, b, c) --> (c == 0 ? a : ((b >> (32 - c)) | (a << c)))
2328///                 --> call llvm.fshl.i32(a, b, c)
2329/// fshr32(a, b, c) --> (c == 0 ? b : ((a >> (32 - c)) | (b << c)))
2330///                 --> call llvm.fshr.i32(a, b, c)
2331static Instruction *foldSelectFunnelShift(SelectInst &Sel,
                                        InstCombiner::BuilderTy &Builder) {
// This must be a power-of-2 type for a bitmasking transform to be valid.
unsigned Width = Sel.getType()->getScalarSizeInBits();
if (!isPowerOf2_32(Width))
  return nullptr;

BinaryOperator *Or0, *Or1;
if (!match(Sel.getFalseValue(), m_OneUse(m_Or(m_BinOp(Or0), m_BinOp(Or1)))))
  return nullptr;

Value *SV0, *SV1, *SA0, *SA1;
if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(SV0),
                                        m_ZExtOrSelf(m_Value(SA0))))) ||
    !match(Or1, m_OneUse(m_LogicalShift(m_Value(SV1),
                                        m_ZExtOrSelf(m_Value(SA1))))) ||
    Or0->getOpcode() == Or1->getOpcode())
  return nullptr;

// Canonicalize to or(shl(SV0, SA0), lshr(SV1, SA1)).
if (Or0->getOpcode() == BinaryOperator::LShr) {
  std::swap(Or0, Or1);
  std::swap(SV0, SV1);
  std::swap(SA0, SA1);
}
assert(Or0->getOpcode() == BinaryOperator::Shl &&(static_cast <bool> (Or0->getOpcode() == BinaryOperator
::Shl && Or1->getOpcode() == BinaryOperator::LShr &&
 "Illegal or(shift,shift) pair") ? void (0) : __assert_fail (
"Or0->getOpcode() == BinaryOperator::Shl && Or1->getOpcode() == BinaryOperator::LShr && \"Illegal or(shift,shift) pair\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 2358
, __extension__ __PRETTY_FUNCTION__))
       Or1->getOpcode() == BinaryOperator::LShr &&(static_cast <bool> (Or0->getOpcode() == BinaryOperator
::Shl && Or1->getOpcode() == BinaryOperator::LShr &&
 "Illegal or(shift,shift) pair") ? void (0) : __assert_fail (
"Or0->getOpcode() == BinaryOperator::Shl && Or1->getOpcode() == BinaryOperator::LShr && \"Illegal or(shift,shift) pair\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 2358
, __extension__ __PRETTY_FUNCTION__))
       "Illegal or(shift,shift) pair")(static_cast <bool> (Or0->getOpcode() == BinaryOperator
::Shl && Or1->getOpcode() == BinaryOperator::LShr &&
 "Illegal or(shift,shift) pair") ? void (0) : __assert_fail (
"Or0->getOpcode() == BinaryOperator::Shl && Or1->getOpcode() == BinaryOperator::LShr && \"Illegal or(shift,shift) pair\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 2358
, __extension__ __PRETTY_FUNCTION__));

// Check the shift amounts to see if they are an opposite pair.
Value *ShAmt;
if (match(SA1, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(SA0)))))
  ShAmt = SA0;
else if (match(SA0, m_OneUse(m_Sub(m_SpecificInt(Width), m_Specific(SA1)))))
  ShAmt = SA1;
else
  return nullptr;

// We should now have this pattern:
// select ?, TVal, (or (shl SV0, SA0), (lshr SV1, SA1))
// The false value of the select must be a funnel-shift of the true value:
// IsFShl -> TVal must be SV0 else TVal must be SV1.
bool IsFshl = (ShAmt == SA0);
Value *TVal = Sel.getTrueValue();
if ((IsFshl && TVal != SV0) || (!IsFshl && TVal != SV1))
  return nullptr;

// Finally, see if the select is filtering out a shift-by-zero.
Value *Cond = Sel.getCondition();
ICmpInst::Predicate Pred;
if (!match(Cond, m_OneUse(m_ICmp(Pred, m_Specific(ShAmt), m_ZeroInt()))) ||
    Pred != ICmpInst::ICMP_EQ)
  return nullptr;

// If this is not a rotate then the select was blocking poison from the
// 'shift-by-zero' non-TVal, but a funnel shift won't - so freeze it.
if (SV0 != SV1) {
  if (IsFshl && !llvm::isGuaranteedNotToBePoison(SV1))
    SV1 = Builder.CreateFreeze(SV1);
  else if (!IsFshl && !llvm::isGuaranteedNotToBePoison(SV0))
    SV0 = Builder.CreateFreeze(SV0);
}

// This is a funnel/rotate that avoids shift-by-bitwidth UB in a suboptimal way.
// Convert to funnel shift intrinsic.
Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
Function *F = Intrinsic::getDeclaration(Sel.getModule(), IID, Sel.getType());
ShAmt = Builder.CreateZExt(ShAmt, Sel.getType());
return CallInst::Create(F, { SV0, SV1, ShAmt });
2400}

2402static Instruction *foldSelectToCopysign(SelectInst &Sel,
                                       InstCombiner::BuilderTy &Builder) {
Value *Cond = Sel.getCondition();
Value *TVal = Sel.getTrueValue();
Value *FVal = Sel.getFalseValue();
Type *SelType = Sel.getType();

// Match select ?, TC, FC where the constants are equal but negated.
// TODO: Generalize to handle a negated variable operand?
const APFloat *TC, *FC;
if (!match(TVal, m_APFloatAllowUndef(TC)) ||
    !match(FVal, m_APFloatAllowUndef(FC)) ||
    !abs(*TC).bitwiseIsEqual(abs(*FC)))
  return nullptr;

assert(TC != FC && "Expected equal select arms to simplify")(static_cast <bool> (TC != FC && "Expected equal select arms to simplify"
) ? void (0) : __assert_fail ("TC != FC && \"Expected equal select arms to simplify\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 2417
, __extension__ __PRETTY_FUNCTION__));

Value *X;
const APInt *C;
bool IsTrueIfSignSet;
ICmpInst::Predicate Pred;
if (!match(Cond, m_OneUse(m_ICmp(Pred, m_BitCast(m_Value(X)), m_APInt(C)))) ||
    !InstCombiner::isSignBitCheck(Pred, *C, IsTrueIfSignSet) ||
    X->getType() != SelType)
  return nullptr;

// If needed, negate the value that will be the sign argument of the copysign:
// (bitcast X) <  0 ? -TC :  TC --> copysign(TC,  X)
// (bitcast X) <  0 ?  TC : -TC --> copysign(TC, -X)
// (bitcast X) >= 0 ? -TC :  TC --> copysign(TC, -X)
// (bitcast X) >= 0 ?  TC : -TC --> copysign(TC,  X)
// Note: FMF from the select can not be propagated to the new instructions.
if (IsTrueIfSignSet ^ TC->isNegative())
  X = Builder.CreateFNeg(X);

// Canonicalize the magnitude argument as the positive constant since we do
// not care about its sign.
Value *MagArg = ConstantFP::get(SelType, abs(*TC));
Function *F = Intrinsic::getDeclaration(Sel.getModule(), Intrinsic::copysign,
                                        Sel.getType());
return CallInst::Create(F, { MagArg, X });
2443}

2445Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) {
if (!isa<VectorType>(Sel.getType()))
  return nullptr;

Value *Cond = Sel.getCondition();
Value *TVal = Sel.getTrueValue();
Value *FVal = Sel.getFalseValue();
Value *C, *X, *Y;

if (match(Cond, m_VecReverse(m_Value(C)))) {
  auto createSelReverse = [&](Value *C, Value *X, Value *Y) {
    Value *V = Builder.CreateSelect(C, X, Y, Sel.getName(), &Sel);
    if (auto *I = dyn_cast<Instruction>(V))
      I->copyIRFlags(&Sel);
    Module *M = Sel.getModule();
    Function *F = Intrinsic::getDeclaration(
        M, Intrinsic::experimental_vector_reverse, V->getType());
    return CallInst::Create(F, V);
  };

  if (match(TVal, m_VecReverse(m_Value(X)))) {
    // select rev(C), rev(X), rev(Y) --> rev(select C, X, Y)
    if (match(FVal, m_VecReverse(m_Value(Y))) &&
        (Cond->hasOneUse() || TVal->hasOneUse() || FVal->hasOneUse()))
      return createSelReverse(C, X, Y);

    // select rev(C), rev(X), FValSplat --> rev(select C, X, FValSplat)
    if ((Cond->hasOneUse() || TVal->hasOneUse()) && isSplatValue(FVal))
      return createSelReverse(C, X, FVal);
  }
  // select rev(C), TValSplat, rev(Y) --> rev(select C, TValSplat, Y)
  else if (isSplatValue(TVal) && match(FVal, m_VecReverse(m_Value(Y))) &&
           (Cond->hasOneUse() || FVal->hasOneUse()))
    return createSelReverse(C, TVal, Y);
}

auto *VecTy = dyn_cast<FixedVectorType>(Sel.getType());
if (!VecTy)
  return nullptr;

unsigned NumElts = VecTy->getNumElements();
APInt UndefElts(NumElts, 0);
APInt AllOnesEltMask(APInt::getAllOnes(NumElts));
if (Value *V = SimplifyDemandedVectorElts(&Sel, AllOnesEltMask, UndefElts)) {
  if (V != &Sel)
    return replaceInstUsesWith(Sel, V);
  return &Sel;
}

// A select of a "select shuffle" with a common operand can be rearranged
// to select followed by "select shuffle". Because of poison, this only works
// in the case of a shuffle with no undefined mask elements.
ArrayRef<int> Mask;
if (match(TVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) &&
    !is_contained(Mask, PoisonMaskElem) &&
    cast<ShuffleVectorInst>(TVal)->isSelect()) {
  if (X == FVal) {
    // select Cond, (shuf_sel X, Y), X --> shuf_sel X, (select Cond, Y, X)
    Value *NewSel = Builder.CreateSelect(Cond, Y, X, "sel", &Sel);
    return new ShuffleVectorInst(X, NewSel, Mask);
  }
  if (Y == FVal) {
    // select Cond, (shuf_sel X, Y), Y --> shuf_sel (select Cond, X, Y), Y
    Value *NewSel = Builder.CreateSelect(Cond, X, Y, "sel", &Sel);
    return new ShuffleVectorInst(NewSel, Y, Mask);
  }
}
if (match(FVal, m_OneUse(m_Shuffle(m_Value(X), m_Value(Y), m_Mask(Mask)))) &&
    !is_contained(Mask, PoisonMaskElem) &&
    cast<ShuffleVectorInst>(FVal)->isSelect()) {
  if (X == TVal) {
    // select Cond, X, (shuf_sel X, Y) --> shuf_sel X, (select Cond, X, Y)
    Value *NewSel = Builder.CreateSelect(Cond, X, Y, "sel", &Sel);
    return new ShuffleVectorInst(X, NewSel, Mask);
  }
  if (Y == TVal) {
    // select Cond, Y, (shuf_sel X, Y) --> shuf_sel (select Cond, Y, X), Y
    Value *NewSel = Builder.CreateSelect(Cond, Y, X, "sel", &Sel);
    return new ShuffleVectorInst(NewSel, Y, Mask);
  }
}

return nullptr;
2528}

2530static Instruction *foldSelectToPhiImpl(SelectInst &Sel, BasicBlock *BB,
                                      const DominatorTree &DT,
                                      InstCombiner::BuilderTy &Builder) {
// Find the block's immediate dominator that ends with a conditional branch
// that matches select's condition (maybe inverted).
auto *IDomNode = DT[BB]->getIDom();
if (!IDomNode)
  return nullptr;
BasicBlock *IDom = IDomNode->getBlock();

Value *Cond = Sel.getCondition();
Value *IfTrue, *IfFalse;
BasicBlock *TrueSucc, *FalseSucc;
if (match(IDom->getTerminator(),
          m_Br(m_Specific(Cond), m_BasicBlock(TrueSucc),
               m_BasicBlock(FalseSucc)))) {
  IfTrue = Sel.getTrueValue();
  IfFalse = Sel.getFalseValue();
} else if (match(IDom->getTerminator(),
                 m_Br(m_Not(m_Specific(Cond)), m_BasicBlock(TrueSucc),
                      m_BasicBlock(FalseSucc)))) {
  IfTrue = Sel.getFalseValue();
  IfFalse = Sel.getTrueValue();
} else
  return nullptr;

// Make sure the branches are actually different.
if (TrueSucc == FalseSucc)
  return nullptr;

// We want to replace select %cond, %a, %b with a phi that takes value %a
// for all incoming edges that are dominated by condition `%cond == true`,
// and value %b for edges dominated by condition `%cond == false`. If %a
// or %b are also phis from the same basic block, we can go further and take
// their incoming values from the corresponding blocks.
BasicBlockEdge TrueEdge(IDom, TrueSucc);
BasicBlockEdge FalseEdge(IDom, FalseSucc);
DenseMap<BasicBlock *, Value *> Inputs;
for (auto *Pred : predecessors(BB)) {
  // Check implication.
  BasicBlockEdge Incoming(Pred, BB);
  if (DT.dominates(TrueEdge, Incoming))
    Inputs[Pred] = IfTrue->DoPHITranslation(BB, Pred);
  else if (DT.dominates(FalseEdge, Incoming))
    Inputs[Pred] = IfFalse->DoPHITranslation(BB, Pred);
  else
    return nullptr;
  // Check availability.
  if (auto *Insn = dyn_cast<Instruction>(Inputs[Pred]))
    if (!DT.dominates(Insn, Pred->getTerminator()))
      return nullptr;
}

Builder.SetInsertPoint(&*BB->begin());
auto *PN = Builder.CreatePHI(Sel.getType(), Inputs.size());
for (auto *Pred : predecessors(BB))
  PN->addIncoming(Inputs[Pred], Pred);
PN->takeName(&Sel);
return PN;
2589}

2591static Instruction *foldSelectToPhi(SelectInst &Sel, const DominatorTree &DT,
                                  InstCombiner::BuilderTy &Builder) {
// Try to replace this select with Phi in one of these blocks.
SmallSetVector<BasicBlock *, 4> CandidateBlocks;
CandidateBlocks.insert(Sel.getParent());
for (Value *V : Sel.operands())
  if (auto *I = dyn_cast<Instruction>(V))
    CandidateBlocks.insert(I->getParent());

for (BasicBlock *BB : CandidateBlocks)
  if (auto *PN = foldSelectToPhiImpl(Sel, BB, DT, Builder))
    return PN;
return nullptr;
2604}

2606static Value *foldSelectWithFrozenICmp(SelectInst &Sel, InstCombiner::BuilderTy &Builder) {
FreezeInst *FI = dyn_cast<FreezeInst>(Sel.getCondition());
if (!FI)
  return nullptr;

Value *Cond = FI->getOperand(0);
Value *TrueVal = Sel.getTrueValue(), *FalseVal = Sel.getFalseValue();

//   select (freeze(x == y)), x, y --> y
//   select (freeze(x != y)), x, y --> x
// The freeze should be only used by this select. Otherwise, remaining uses of
// the freeze can observe a contradictory value.
//   c = freeze(x == y)   ; Let's assume that y = poison & x = 42; c is 0 or 1
//   a = select c, x, y   ;
//   f(a, c)              ; f(poison, 1) cannot happen, but if a is folded
//                        ; to y, this can happen.
CmpInst::Predicate Pred;
if (FI->hasOneUse() &&
    match(Cond, m_c_ICmp(Pred, m_Specific(TrueVal), m_Specific(FalseVal))) &&
    (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE)) {
  return Pred == ICmpInst::ICMP_EQ ? FalseVal : TrueVal;
}

return nullptr;
2630}

2632Instruction *InstCombinerImpl::foldAndOrOfSelectUsingImpliedCond(Value *Op,
                                                               SelectInst &SI,
                                                               bool IsAnd) {
Value *CondVal = SI.getCondition();
Value *A = SI.getTrueValue();
Value *B = SI.getFalseValue();

assert(Op->getType()->isIntOrIntVectorTy(1) &&(static_cast <bool> (Op->getType()->isIntOrIntVectorTy
(1) && "Op must be either i1 or vector of i1.") ? void
 (0) : __assert_fail ("Op->getType()->isIntOrIntVectorTy(1) && \"Op must be either i1 or vector of i1.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 2640
, __extension__ __PRETTY_FUNCTION__))
       "Op must be either i1 or vector of i1.")(static_cast <bool> (Op->getType()->isIntOrIntVectorTy
(1) && "Op must be either i1 or vector of i1.") ? void
 (0) : __assert_fail ("Op->getType()->isIntOrIntVectorTy(1) && \"Op must be either i1 or vector of i1.\""
, "llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp", 2640
, __extension__ __PRETTY_FUNCTION__));

std::optional<bool> Res = isImpliedCondition(Op, CondVal, DL, IsAnd);
if (!Res)
  return nullptr;

Value *Zero = Constant::getNullValue(A->getType());
Value *One = Constant::getAllOnesValue(A->getType());

if (*Res == true) {
  if (IsAnd)
    // select op, (select cond, A, B), false => select op, A, false
    // and    op, (select cond, A, B)        => select op, A, false
    //   if op = true implies condval = true.
    return SelectInst::Create(Op, A, Zero);
  else
    // select op, true, (select cond, A, B) => select op, true, A
    // or     op, (select cond, A, B)       => select op, true, A
    //   if op = false implies condval = true.
    return SelectInst::Create(Op, One, A);
} else {
  if (IsAnd)
    // select op, (select cond, A, B), false => select op, B, false
    // and    op, (select cond, A, B)        => select op, B, false
    //   if op = true implies condval = false.
    return SelectInst::Create(Op, B, Zero);
  else
    // select op, true, (select cond, A, B) => select op, true, B
    // or     op, (select cond, A, B)       => select op, true, B
    //   if op = false implies condval = false.
    return SelectInst::Create(Op, One, B);
}
2672}

2674// Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need
2675// fast-math-flags (nsz) or fsub with +0.0 (not fneg) for this to work.
2676static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
                                           InstCombinerImpl &IC) {
Value *CondVal = SI.getCondition();

bool ChangedFMF = false;
for (bool Swap : {false, true}) {
  Value *TrueVal = SI.getTrueValue();
  Value *X = SI.getFalseValue();
  CmpInst::Predicate Pred;

  if (Swap)
    std::swap(TrueVal, X);

  if (!match(CondVal, m_FCmp(Pred, m_Specific(X), m_AnyZeroFP())))
    continue;

  // fold (X <= +/-0.0) ? (0.0 - X) : X to fabs(X), when 'Swap' is false
  // fold (X >  +/-0.0) ? X : (0.0 - X) to fabs(X), when 'Swap' is true
  if (match(TrueVal, m_FSub(m_PosZeroFP(), m_Specific(X)))) {
    if (!Swap && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) {
      Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
      return IC.replaceInstUsesWith(SI, Fabs);
    }
    if (Swap && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT)) {
      Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
      return IC.replaceInstUsesWith(SI, Fabs);
    }
  }

  if (!match(TrueVal, m_FNeg(m_Specific(X))))
    return nullptr;

  // Forward-propagate nnan and ninf from the fneg to the select.
  // If all inputs are not those values, then the select is not either.
  // Note: nsz is defined differently, so it may not be correct to propagate.
  FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags();
  if (FMF.noNaNs() && !SI.hasNoNaNs()) {
    SI.setHasNoNaNs(true);
    ChangedFMF = true;
  }
  if (FMF.noInfs() && !SI.hasNoInfs()) {
    SI.setHasNoInfs(true);
    ChangedFMF = true;
  }

  // With nsz, when 'Swap' is false:
  // fold (X < +/-0.0) ? -X : X or (X <= +/-0.0) ? -X : X to fabs(X)
  // fold (X > +/-0.0) ? -X : X or (X >= +/-0.0) ? -X : X to -fabs(x)
  // when 'Swap' is true:
  // fold (X > +/-0.0) ? X : -X or (X >= +/-0.0) ? X : -X to fabs(X)
  // fold (X < +/-0.0) ? X : -X or (X <= +/-0.0) ? X : -X to -fabs(X)
  //
  // Note: We require "nnan" for this fold because fcmp ignores the signbit
  //       of NAN, but IEEE-754 specifies the signbit of NAN values with
  //       fneg/fabs operations.
  if (!SI.hasNoSignedZeros() || !SI.hasNoNaNs())
    return nullptr;

  if (Swap)
    Pred = FCmpInst::getSwappedPredicate(Pred);

  bool IsLTOrLE = Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
                  Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE;
  bool IsGTOrGE = Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE ||
                  Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE;

  if (IsLTOrLE) {
    Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
    return IC.replaceInstUsesWith(SI, Fabs);
  }
  if (IsGTOrGE) {
    Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
    Instruction *NewFNeg = UnaryOperator::CreateFNeg(Fabs);
    NewFNeg->setFastMathFlags(SI.getFastMathFlags());
    return NewFNeg;
  }
}

return ChangedFMF ? &SI : nullptr;
2755}

2757// Match the following IR pattern:
2758//   %x.lowbits = and i8 %x, %lowbitmask
2759//   %x.lowbits.are.zero = icmp eq i8 %x.lowbits, 0
2760//   %x.biased = add i8 %x, %bias
2761//   %x.biased.highbits = and i8 %x.biased, %highbitmask
2762//   %x.roundedup = select i1 %x.lowbits.are.zero, i8 %x, i8 %x.biased.highbits
2763// Define:
2764//   %alignment = add i8 %lowbitmask, 1
2765// Iff 1. an %alignment is a power-of-two (aka, %lowbitmask is a low bit mask)
2766// and 2. %bias is equal to either %lowbitmask or %alignment,
2767// and 3. %highbitmask is equal to ~%lowbitmask (aka, to -%alignment)
2768// then this pattern can be transformed into:
2769//   %x.offset = add i8 %x, %lowbitmask
2770//   %x.roundedup = and i8 %x.offset, %highbitmask
2771static Value *
2772foldRoundUpIntegerWithPow2Alignment(SelectInst &SI,
                                  InstCombiner::BuilderTy &Builder) {
Value *Cond = SI.getCondition();
Value *X = SI.getTrueValue();
Value *XBiasedHighBits = SI.getFalseValue();

ICmpInst::Predicate Pred;
Value *XLowBits;
if (!match(Cond, m_ICmp(Pred, m_Value(XLowBits), m_ZeroInt())) ||
    !ICmpInst::isEquality(Pred))
  return nullptr;

if (Pred == ICmpInst::Predicate::ICMP_NE)
  std::swap(X, XBiasedHighBits);

// FIXME: we could support non non-splats here.

const APInt *LowBitMaskCst;
if (!match(XLowBits, m_And(m_Specific(X), m_APIntAllowUndef(LowBitMaskCst))))
  return nullptr;

// Match even if the AND and ADD are swapped.
const APInt *BiasCst, *HighBitMaskCst;
if (!match(XBiasedHighBits,
           m_And(m_Add(m_Specific(X), m_APIntAllowUndef(BiasCst)),
                 m_APIntAllowUndef(HighBitMaskCst))) &&
    !match(XBiasedHighBits,
           m_Add(m_And(m_Specific(X), m_APIntAllowUndef(HighBitMaskCst)),
                 m_APIntAllowUndef(BiasCst))))
  return nullptr;

if (!LowBitMaskCst->isMask())
  return nullptr;

APInt InvertedLowBitMaskCst = ~*LowBitMaskCst;
if (InvertedLowBitMaskCst != *HighBitMaskCst)
  return nullptr;

APInt AlignmentCst = *LowBitMaskCst + 1;

if (*BiasCst != AlignmentCst && *BiasCst != *LowBitMaskCst)
  return nullptr;

if (!XBiasedHighBits->hasOneUse()) {
  if (*BiasCst == *LowBitMaskCst)
    return XBiasedHighBits;
  return nullptr;
}

// FIXME: could we preserve undef's here?
Type *Ty = X->getType();
Value *XOffset = Builder.CreateAdd(X, ConstantInt::get(Ty, *LowBitMaskCst),
                                   X->getName() + ".biased");
Value *R = Builder.CreateAnd(XOffset, ConstantInt::get(Ty, *HighBitMaskCst));
R->takeName(&SI);
return R;
2828}

2830namespace {
2831struct DecomposedSelect {
Value *Cond = nullptr;
47
←
Null pointer value stored to 'OuterSel.Cond'→
Value *TrueVal = nullptr;
Value *FalseVal = nullptr;
2835};
2836} // namespace

2838/// Look for patterns like
2839///   %outer.cond = select i1 %inner.cond, i1 %alt.cond, i1 false
2840///   %inner.sel = select i1 %inner.cond, i8 %inner.sel.t, i8 %inner.sel.f
2841///   %outer.sel = select i1 %outer.cond, i8 %outer.sel.t, i8 %inner.sel
2842/// and rewrite it as
2843///   %inner.sel = select i1 %cond.alternative, i8 %sel.outer.t, i8 %sel.inner.t
2844///   %sel.outer = select i1 %cond.inner, i8 %inner.sel, i8 %sel.inner.f
2845static Instruction *foldNestedSelects(SelectInst &OuterSelVal,
                                    InstCombiner::BuilderTy &Builder) {
// We must start with a `select`.
DecomposedSelect OuterSel;
46
←
Calling implicit default constructor for 'DecomposedSelect'→
48
←
Returning from default constructor for 'DecomposedSelect'→
match(&OuterSelVal,
57
←
Calling 'match<llvm::SelectInst, llvm::PatternMatch::ThreeOps_match<llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>, 57>>'→
59
←
Returning from 'match<llvm::SelectInst, llvm::PatternMatch::ThreeOps_match<llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>, 57>>'→
      m_Select(m_Value(OuterSel.Cond), m_Value(OuterSel.TrueVal),
49
←
Calling 'm_Value'→
53
←
Returning from 'm_Value'→
54
←
Calling 'm_Select<llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>>'→
56
←
Returning from 'm_Select<llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>, llvm::PatternMatch::bind_ty<llvm::Value>>'→
               m_Value(OuterSel.FalseVal)));

// Canonicalize inversion of the outermost `select`'s condition.
if (match(OuterSel.Cond, m_Not(m_Value(OuterSel.Cond))))
60
←
Calling 'm_Value'→
64
←
Returning from 'm_Value'→
65
←
Calling 'm_Not<llvm::PatternMatch::bind_ty<llvm::Value>>'→
70
←
Returning from 'm_Not<llvm::PatternMatch::bind_ty<llvm::Value>>'→
71
←
Passing null pointer value via 1st parameter 'V'→
72
←
Calling 'match<llvm::Value, llvm::PatternMatch::BinaryOp_match<llvm::PatternMatch::cstval_pred_ty<llvm::PatternMatch::is_all_ones, llvm::ConstantInt>, llvm::PatternMatch::bind_ty<llvm::Value>, 30, true>>'→
  std::swap(OuterSel.TrueVal, OuterSel.FalseVal);

// The condition of the outermost select must be an `and`/`or`.
if (!match(OuterSel.Cond, m_c_LogicalOp(m_Value(), m_Value())))
  return nullptr;

// Depending on the logical op, inner select might be in different hand.
bool IsAndVariant = match(OuterSel.Cond, m_LogicalAnd());
Value *InnerSelVal = IsAndVariant ? OuterSel.FalseVal : OuterSel.TrueVal;

// Profitability check - avoid increasing instruction count.
if (none_of(ArrayRef<Value *>({OuterSelVal.getCondition(), InnerSelVal}),
            [](Value *V) { return V->hasOneUse(); }))
  return nullptr;

// The appropriate hand of the outermost `select` must be a select itself.
DecomposedSelect InnerSel;
if (!match(InnerSelVal,
           m_Select(m_Value(InnerSel.Cond), m_Value(InnerSel.TrueVal),
                    m_Value(InnerSel.FalseVal))))
  return nullptr;

// Canonicalize inversion of the innermost `select`'s condition.
if (match(InnerSel.Cond, m_Not(m_Value(InnerSel.Cond))))
  std::swap(InnerSel.TrueVal, InnerSel.FalseVal);

Value *AltCond = nullptr;
auto matchOuterCond = [OuterSel, &AltCond](auto m_InnerCond) {
  return match(OuterSel.Cond, m_c_LogicalOp(m_InnerCond, m_Value(AltCond)));
};

// Finally, match the condition that was driving the outermost `select`,
// it should be a logical operation between the condition that was driving
// the innermost `select` (after accounting for the possible inversions
// of the condition), and some other condition.
if (matchOuterCond(m_Specific(InnerSel.Cond))) {
  // Done!
} else if (Value * NotInnerCond; matchOuterCond(m_CombineAnd(
               m_Not(m_Specific(InnerSel.Cond)), m_Value(NotInnerCond)))) {
  // Done!
  std::swap(InnerSel.TrueVal, InnerSel.FalseVal);
  InnerSel.Cond = NotInnerCond;
} else // Not the pattern we were looking for.
  return nullptr;

Value *SelInner = Builder.CreateSelect(
    AltCond, IsAndVariant ? OuterSel.TrueVal : InnerSel.FalseVal,
    IsAndVariant ? InnerSel.TrueVal : OuterSel.FalseVal);
SelInner->takeName(InnerSelVal);
return SelectInst::Create(InnerSel.Cond,
                          IsAndVariant ? SelInner : InnerSel.TrueVal,
                          !IsAndVariant ? SelInner : InnerSel.FalseVal);
2907}

2909Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
Value *CondVal = SI.getCondition();
Value *TrueVal = SI.getTrueValue();
Value *FalseVal = SI.getFalseValue();
Type *SelType = SI.getType();

// Avoid potential infinite loops by checking for non-constant condition.
// TODO: Can we assert instead by improving canonicalizeSelectToShuffle()?
//       Scalar select must have simplified?
if (!SelType->isIntOrIntVectorTy(1) || isa<Constant>(CondVal) ||
    TrueVal->getType() != CondVal->getType())
  return nullptr;

auto *One = ConstantInt::getTrue(SelType);
auto *Zero = ConstantInt::getFalse(SelType);
Value *A, *B, *C, *D;

// Folding select to and/or i1 isn't poison safe in general. impliesPoison
// checks whether folding it does not convert a well-defined value into
// poison.
if (match(TrueVal, m_One())) {
  if (impliesPoison(FalseVal, CondVal)) {
    // Change: A = select B, true, C --> A = or B, C
    return BinaryOperator::CreateOr(CondVal, FalseVal);
  }

  if (auto *LHS = dyn_cast<FCmpInst>(CondVal))
    if (auto *RHS = dyn_cast<FCmpInst>(FalseVal))
      if (Value *V = foldLogicOfFCmps(LHS, RHS, /*IsAnd*/ false,
                                      /*IsSelectLogical*/ true))
        return replaceInstUsesWith(SI, V);

  // (A && B) || (C && B) --> (A || C) && B
  if (match(CondVal, m_LogicalAnd(m_Value(A), m_Value(B))) &&
      match(FalseVal, m_LogicalAnd(m_Value(C), m_Value(D))) &&
      (CondVal->hasOneUse() || FalseVal->hasOneUse())) {
    bool CondLogicAnd = isa<SelectInst>(CondVal);
    bool FalseLogicAnd = isa<SelectInst>(FalseVal);
    auto AndFactorization = [&](Value *Common, Value *InnerCond,
                                Value *InnerVal,
                                bool SelFirst = false) -> Instruction * {
      Value *InnerSel = Builder.CreateSelect(InnerCond, One, InnerVal);
      if (SelFirst)
        std::swap(Common, InnerSel);
      if (FalseLogicAnd || (CondLogicAnd && Common == A))
        return SelectInst::Create(Common, InnerSel, Zero);
      else
        return BinaryOperator::CreateAnd(Common, InnerSel);
    };

    if (A == C)
      return AndFactorization(A, B, D);
    if (A == D)
      return AndFactorization(A, B, C);
    if (B == C)
      return AndFactorization(B, A, D);
    if (B == D)
      return AndFactorization(B, A, C, CondLogicAnd && FalseLogicAnd);
  }
}

if (match(FalseVal, m_Zero())) {
  if (impliesPoison(TrueVal, CondVal)) {
    // Change: A = select B, C, false --> A = and B, C
    return BinaryOperator::CreateAnd(CondVal, TrueVal);
  }

  if (auto *LHS = dyn_cast<FCmpInst>(CondVal))
    if (auto *RHS = dyn_cast<FCmpInst>(TrueVal))
      if (Value *V = foldLogicOfFCmps(LHS, RHS, /*IsAnd*/ true,
                                      /*IsSelectLogical*/ true))
        return replaceInstUsesWith(SI, V);

  // (A || B) && (C || B) --> (A && C) || B
  if (match(CondVal, m_LogicalOr(m_Value(A), m_Value(B))) &&
      match(TrueVal, m_LogicalOr(m_Value(C), m_Value(D))) &&
      (CondVal->hasOneUse() || TrueVal->hasOneUse())) {
    bool CondLogicOr = isa<SelectInst>(CondVal);
    bool TrueLogicOr = isa<SelectInst>(TrueVal);
    auto OrFactorization = [&](Value *Common, Value *InnerCond,
                               Value *InnerVal,
                               bool SelFirst = false) -> Instruction * {
      Value *InnerSel = Builder.CreateSelect(InnerCond, InnerVal, Zero);
      if (SelFirst)
        std::swap(Common, InnerSel);
      if (TrueLogicOr || (CondLogicOr && Common == A))
        return SelectInst::Create(Common, One, InnerSel);
      else
        return BinaryOperator::CreateOr(Common, InnerSel);
    };

    if (A == C)
      return OrFactorization(A, B, D);
    if (A == D)
      return OrFactorization(A, B, C);
    if (B == C)
      return OrFactorization(B, A, D);
    if (B == D)
      return OrFactorization(B, A, C, CondLogicOr && TrueLogicOr);
  }
}

// We match the "full" 0 or 1 constant here to avoid a potential infinite
// loop with vectors that may have undefined/poison elements.
// select a, false, b -> select !a, b, false
if (match(TrueVal, m_Specific(Zero))) {
  Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName());
  return SelectInst::Create(NotCond, FalseVal, Zero);
}
// select a, b, true -> select !a, true, b
if (match(FalseVal, m_Specific(One))) {
  Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName());
  return SelectInst::Create(NotCond, One, TrueVal);
}

// DeMorgan in select form: !a && !b --> !(a || b)
// select !a, !b, false --> not (select a, true, b)
if (match(&SI, m_LogicalAnd(m_Not(m_Value(A)), m_Not(m_Value(B)))) &&
    (CondVal->hasOneUse() || TrueVal->hasOneUse()) &&
    !match(A, m_ConstantExpr()) && !match(B, m_ConstantExpr()))
  return BinaryOperator::CreateNot(Builder.CreateSelect(A, One, B));

// DeMorgan in select form: !a || !b --> !(a && b)
// select !a, true, !b --> not (select a, b, false)
if (match(&SI, m_LogicalOr(m_Not(m_Value(A)), m_Not(m_Value(B)))) &&
    (CondVal->hasOneUse() || FalseVal->hasOneUse()) &&
    !match(A, m_ConstantExpr()) && !match(B, m_ConstantExpr()))
  return BinaryOperator::CreateNot(Builder.CreateSelect(A, B, Zero));

// select (select a, true, b), true, b -> select a, true, b
if (match(CondVal, m_Select(m_Value(A), m_One(), m_Value(B))) &&
    match(TrueVal, m_One()) && match(FalseVal, m_Specific(B)))
  return replaceOperand(SI, 0, A);
// select (select a, b, false), b, false -> select a, b, false
if (match(CondVal, m_Select(m_Value(A), m_Value(B), m_Zero())) &&
    match(TrueVal, m_Specific(B)) && match(FalseVal, m_Zero()))
  return replaceOperand(SI, 0, A);
// select a, (select ~a, true, b), false -> select a, b, false
if (match(TrueVal, m_c_LogicalOr(m_Not(m_Specific(CondVal)), m_Value(B))) &&
    match(FalseVal, m_Zero()))
  return replaceOperand(SI, 1, B);
// select a, true, (select ~a, b, false) -> select a, true, b
if (match(FalseVal, m_c_LogicalAnd(m_Not(m_Specific(CondVal)), m_Value(B))) &&
    match(TrueVal, m_One()))
  return replaceOperand(SI, 2, B);

// ~(A & B) & (A | B) --> A ^ B
if (match(&SI, m_c_LogicalAnd(m_Not(m_LogicalAnd(m_Value(A), m_Value(B))),
                              m_c_LogicalOr(m_Deferred(A), m_Deferred(B)))))
  return BinaryOperator::CreateXor(A, B);

// select (~a | c), a, b -> and a, (or c, freeze(b))
if (match(CondVal, m_c_Or(m_Not(m_Specific(TrueVal)), m_Value(C))) &&
    CondVal->hasOneUse()) {
  FalseVal = Builder.CreateFreeze(FalseVal);
  return BinaryOperator::CreateAnd(TrueVal, Builder.CreateOr(C, FalseVal));
}
// select (~c & b), a, b -> and b, (or freeze(a), c)
if (match(CondVal, m_c_And(m_Not(m_Value(C)), m_Specific(FalseVal))) &&
    CondVal->hasOneUse()) {
  TrueVal = Builder.CreateFreeze(TrueVal);
  return BinaryOperator::CreateAnd(FalseVal, Builder.CreateOr(C, TrueVal));
}

if (match(FalseVal, m_Zero()) || match(TrueVal, m_One())) {
  Use *Y = nullptr;
  bool IsAnd = match(FalseVal, m_Zero()) ? true : false;
  Value *Op1 = IsAnd ? TrueVal : FalseVal;
  if (isCheckForZeroAndMulWithOverflow(CondVal, Op1, IsAnd, Y)) {
    auto *FI = new FreezeInst(*Y, (*Y)->getName() + ".fr");
    InsertNewInstBefore(FI, *cast<Instruction>(Y->getUser()));
    replaceUse(*Y, FI);
    return replaceInstUsesWith(SI, Op1);
  }

  if (auto *Op1SI = dyn_cast<SelectInst>(Op1))
    if (auto *I = foldAndOrOfSelectUsingImpliedCond(CondVal, *Op1SI,
                                                    /* IsAnd */ IsAnd))
      return I;

  if (auto *ICmp0 = dyn_cast<ICmpInst>(CondVal))
    if (auto *ICmp1 = dyn_cast<ICmpInst>(Op1))
      if (auto *V = foldAndOrOfICmps(ICmp0, ICmp1, SI, IsAnd,
                                     /* IsLogical */ true))
        return replaceInstUsesWith(SI, V);
}

// select (a || b), c, false -> select a, c, false
// select c, (a || b), false -> select c, a, false
//   if c implies that b is false.
if (match(CondVal, m_LogicalOr(m_Value(A), m_Value(B))) &&
    match(FalseVal, m_Zero())) {
  std::optional<bool> Res = isImpliedCondition(TrueVal, B, DL);
  if (Res && *Res == false)
    return replaceOperand(SI, 0, A);
}
if (match(TrueVal, m_LogicalOr(m_Value(A), m_Value(B))) &&
    match(FalseVal, m_Zero())) {
  std::optional<bool> Res = isImpliedCondition(CondVal, B, DL);
  if (Res && *Res == false)
    return replaceOperand(SI, 1, A);
}
// select c, true, (a && b)  -> select c, true, a
// select (a && b), true, c  -> select a, true, c
//   if c = false implies that b = true
if (match(TrueVal, m_One()) &&
    match(FalseVal, m_LogicalAnd(m_Value(A), m_Value(B)))) {
  std::optional<bool> Res = isImpliedCondition(CondVal, B, DL, false);
  if (Res && *Res == true)
    return replaceOperand(SI, 2, A);
}
if (match(CondVal, m_LogicalAnd(m_Value(A), m_Value(B))) &&
    match(TrueVal, m_One())) {
  std::optional<bool> Res = isImpliedCondition(FalseVal, B, DL, false);
  if (Res && *Res == true)
    return replaceOperand(SI, 0, A);
}

if (match(TrueVal, m_One())) {
  Value *C;

  // (C && A) || (!C && B) --> sel C, A, B
  // (A && C) || (!C && B) --> sel C, A, B
  // (C && A) || (B && !C) --> sel C, A, B
  // (A && C) || (B && !C) --> sel C, A, B (may require freeze)
  if (match(FalseVal, m_c_LogicalAnd(m_Not(m_Value(C)), m_Value(B))) &&
      match(CondVal, m_c_LogicalAnd(m_Specific(C), m_Value(A)))) {
    auto *SelCond = dyn_cast<SelectInst>(CondVal);
    auto *SelFVal = dyn_cast<SelectInst>(FalseVal);
    bool MayNeedFreeze = SelCond && SelFVal &&
                         match(SelFVal->getTrueValue(),
                               m_Not(m_Specific(SelCond->getTrueValue())));
    if (MayNeedFreeze)
      C = Builder.CreateFreeze(C);
    return SelectInst::Create(C, A, B);
  }

  // (!C && A) || (C && B) --> sel C, B, A
  // (A && !C) || (C && B) --> sel C, B, A
  // (!C && A) || (B && C) --> sel C, B, A
  // (A && !C) || (B && C) --> sel C, B, A (may require freeze)
  if (match(CondVal, m_c_LogicalAnd(m_Not(m_Value(C)), m_Value(A))) &&
      match(FalseVal, m_c_LogicalAnd(m_Specific(C), m_Value(B)))) {
    auto *SelCond = dyn_cast<SelectInst>(CondVal);
    auto *SelFVal = dyn_cast<SelectInst>(FalseVal);
    bool MayNeedFreeze = SelCond && SelFVal &&
                         match(SelCond->getTrueValue(),
                               m_Not(m_Specific(SelFVal->getTrueValue())));
    if (MayNeedFreeze)
      C = Builder.CreateFreeze(C);
    return SelectInst::Create(C, B, A);
  }
}

return nullptr;
3164}

3166// Return true if we can safely remove the select instruction for std::bit_ceil
3167// pattern.
3168static bool isSafeToRemoveBitCeilSelect(ICmpInst::Predicate Pred, Value *Cond0,
                                      const APInt *Cond1, Value *CtlzOp,
                                      unsigned BitWidth) {
// The challenge in recognizing std::bit_ceil(X) is that the operand is used
// for the CTLZ proper and select condition, each possibly with some
// operation like add and sub.
//
// Our aim is to make sure that -ctlz & (BitWidth - 1) == 0 even when the
// select instruction would select 1, which allows us to get rid of the select
// instruction.
//
// To see if we can do so, we do some symbolic execution with ConstantRange.
// Specifically, we compute the range of values that Cond0 could take when
// Cond == false.  Then we successively transform the range until we obtain
// the range of values that CtlzOp could take.
//
// Conceptually, we follow the def-use chain backward from Cond0 while
// transforming the range for Cond0 until we meet the common ancestor of Cond0
// and CtlzOp.  Then we follow the def-use chain forward until we obtain the
// range for CtlzOp.  That said, we only follow at most one ancestor from
// Cond0.  Likewise, we only follow at most one ancestor from CtrlOp.

ConstantRange CR = ConstantRange::makeExactICmpRegion(
    CmpInst::getInversePredicate(Pred), *Cond1);

// Match the operation that's used to compute CtlzOp from CommonAncestor.  If
// CtlzOp == CommonAncestor, return true as no operation is needed.  If a
// match is found, execute the operation on CR, update CR, and return true.
// Otherwise, return false.
auto MatchForward = [&](Value *CommonAncestor) {
  const APInt *C = nullptr;
  if (CtlzOp == CommonAncestor)
    return true;
  if (match(CtlzOp, m_Add(m_Specific(CommonAncestor), m_APInt(C)))) {
    CR = CR.add(*C);
    return true;
  }
  if (match(CtlzOp, m_Sub(m_APInt(C), m_Specific(CommonAncestor)))) {
    CR = ConstantRange(*C).sub(CR);
    return true;
  }
  if (match(CtlzOp, m_Not(m_Specific(CommonAncestor)))) {
    CR = CR.binaryNot();
    return true;
  }
  return false;
};

const APInt *C = nullptr;
Value *CommonAncestor;
if (MatchForward(Cond0)) {
  // Cond0 is either CtlzOp or CtlzOp's parent.  CR has been updated.
} else if (match(Cond0, m_Add(m_Value(CommonAncestor), m_APInt(C)))) {
  CR = CR.sub(*C);
  if (!MatchForward(CommonAncestor))
    return false;
  // Cond0's parent is either CtlzOp or CtlzOp's parent.  CR has been updated.
} else {
  return false;
}

// Return true if all the values in the range are either 0 or negative (if
// treated as signed).  We do so by evaluating:
//
//   CR - 1 u>= (1 << BitWidth) - 1.
APInt IntMax = APInt::getSignMask(BitWidth) - 1;
CR = CR.sub(APInt(BitWidth, 1));
return CR.icmp(ICmpInst::ICMP_UGE, IntMax);
3236}

3238// Transform the std::bit_ceil(X) pattern like:
3239//
3240//   %dec = add i32 %x, -1
3241//   %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
3242//   %sub = sub i32 32, %ctlz
3243//   %shl = shl i32 1, %sub
3244//   %ugt = icmp ugt i32 %x, 1
3245//   %sel = select i1 %ugt, i32 %shl, i32 1
3246//
3247// into:
3248//
3249//   %dec = add i32 %x, -1
3250//   %ctlz = tail call i32 @llvm.ctlz.i32(i32 %dec, i1 false)
3251//   %neg = sub i32 0, %ctlz
3252//   %masked = and i32 %ctlz, 31
3253//   %shl = shl i32 1, %sub
3254//
3255// Note that the select is optimized away while the shift count is masked with
3256// 31.  We handle some variations of the input operand like std::bit_ceil(X +
3257// 1).
3258static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder) {
Type *SelType = SI.getType();
unsigned BitWidth = SelType->getScalarSizeInBits();

Value *FalseVal = SI.getFalseValue();
Value *TrueVal = SI.getTrueValue();
ICmpInst::Predicate Pred;
const APInt *Cond1;
Value *Cond0, *Ctlz, *CtlzOp;
if (!match(SI.getCondition(), m_ICmp(Pred, m_Value(Cond0), m_APInt(Cond1))))
  return nullptr;

if (match(TrueVal, m_One())) {
  std::swap(FalseVal, TrueVal);
  Pred = CmpInst::getInversePredicate(Pred);
}

if (!match(FalseVal, m_One()) ||
    !match(TrueVal,
           m_OneUse(m_Shl(m_One(), m_OneUse(m_Sub(m_SpecificInt(BitWidth),
                                                  m_Value(Ctlz)))))) ||
    !match(Ctlz, m_Intrinsic<Intrinsic::ctlz>(m_Value(CtlzOp), m_Zero())) ||
    !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp, BitWidth))
  return nullptr;

// Build 1 << (-CTLZ & (BitWidth-1)).  The negation likely corresponds to a
// single hardware instruction as opposed to BitWidth - CTLZ, where BitWidth
// is an integer constant.  Masking with BitWidth-1 comes free on some
// hardware as part of the shift instruction.
Value *Neg = Builder.CreateNeg(Ctlz);
Value *Masked =
    Builder.CreateAnd(Neg, ConstantInt::get(SelType, BitWidth - 1));
return BinaryOperator::Create(Instruction::Shl, ConstantInt::get(SelType, 1),
                              Masked);
3292}

3294Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
Value *CondVal = SI.getCondition();
Value *TrueVal = SI.getTrueValue();
Value *FalseVal = SI.getFalseValue();
Type *SelType = SI.getType();

if (Value *V = simplifySelectInst(CondVal, TrueVal, FalseVal,
1
Assuming 'V' is null→
2
←
Taking false branch→
                                  SQ.getWithInstruction(&SI)))
  return replaceInstUsesWith(SI, V);

if (Instruction *I2.1
'I' is null
1
'I' is null
 = canonicalizeSelectToShuffle(SI))
3
←
Taking false branch→
  return I;

if (Instruction *I3.1
'I' is null
1
'I' is null
 = canonicalizeScalarSelectOfVecs(SI, *this))
4
←
Taking false branch→
  return I;

// If the type of select is not an integer type or if the condition and
// the selection type are not both scalar nor both vector types, there is no
// point in attempting to match these patterns.
Type *CondType = CondVal->getType();
if (!isa<Constant>(CondVal) && SelType->isIntOrIntVectorTy() &&
5
←
Assuming 'CondVal' is not a 'Constant'→
    CondType->isVectorTy() == SelType->isVectorTy()) {
  if (Value *S = simplifyWithOpReplaced(TrueVal, CondVal,
                                        ConstantInt::getTrue(CondType), SQ,
                                        /* AllowRefinement */ true))
    return replaceOperand(SI, 1, S);

  if (Value *S = simplifyWithOpReplaced(FalseVal, CondVal,
                                        ConstantInt::getFalse(CondType), SQ,
                                        /* AllowRefinement */ true))
    return replaceOperand(SI, 2, S);

  // Handle patterns involving sext/zext + not explicitly,
  // as simplifyWithOpReplaced() only looks past one instruction.
  Value *NotCond;

  // select a, sext(!a), b -> select !a, b, 0
  // select a, zext(!a), b -> select !a, b, 0
  if (match(TrueVal, m_ZExtOrSExt(m_CombineAnd(m_Value(NotCond),
                                               m_Not(m_Specific(CondVal))))))
    return SelectInst::Create(NotCond, FalseVal,
                              Constant::getNullValue(SelType));

  // select a, b, zext(!a) -> select !a, 1, b
  if (match(FalseVal, m_ZExt(m_CombineAnd(m_Value(NotCond),
                                          m_Not(m_Specific(CondVal))))))
    return SelectInst::Create(NotCond, ConstantInt::get(SelType, 1), TrueVal);

  // select a, b, sext(!a) -> select !a, -1, b
  if (match(FalseVal, m_SExt(m_CombineAnd(m_Value(NotCond),
                                          m_Not(m_Specific(CondVal))))))
    return SelectInst::Create(NotCond, Constant::getAllOnesValue(SelType),
                              TrueVal);
}

if (Instruction *R = foldSelectOfBools(SI))
6
←
Assuming 'R' is null→
  return R;

// Selecting between two integer or vector splat integer constants?
//
// Note that we don't handle a scalar select of vectors:
// select i1 %c, <2 x i8> <1, 1>, <2 x i8> <0, 0>
// because that may need 3 instructions to splat the condition value:
// extend, insertelement, shufflevector.
//
// Do not handle i1 TrueVal and FalseVal otherwise would result in
// zext/sext i1 to i1.
if (SelType->isIntOrIntVectorTy() && !SelType->isIntOrIntVectorTy(1) &&
    CondVal->getType()->isVectorTy() == SelType->isVectorTy()) {
  // select C, 1, 0 -> zext C to int
  if (match(TrueVal, m_One()) && match(FalseVal, m_Zero()))
    return new ZExtInst(CondVal, SelType);

  // select C, -1, 0 -> sext C to int
  if (match(TrueVal, m_AllOnes()) && match(FalseVal, m_Zero()))
    return new SExtInst(CondVal, SelType);

  // select C, 0, 1 -> zext !C to int
  if (match(TrueVal, m_Zero()) && match(FalseVal, m_One())) {
    Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName());
    return new ZExtInst(NotCond, SelType);
  }

  // select C, 0, -1 -> sext !C to int
  if (match(TrueVal, m_Zero()) && match(FalseVal, m_AllOnes())) {
    Value *NotCond = Builder.CreateNot(CondVal, "not." + CondVal->getName());
    return new SExtInst(NotCond, SelType);
  }
}

if (auto *FCmp7.1
'FCmp' is null
1
'FCmp' is null
 = dyn_cast<FCmpInst>(CondVal)) {
7
←
Assuming 'CondVal' is not a 'CastReturnType'→
8
←
Taking false branch→
  Value *Cmp0 = FCmp->getOperand(0), *Cmp1 = FCmp->getOperand(1);
  // Are we selecting a value based on a comparison of the two values?
  if ((Cmp0 == TrueVal && Cmp1 == FalseVal) ||
      (Cmp0 == FalseVal && Cmp1 == TrueVal)) {
    // Canonicalize to use ordered comparisons by swapping the select
    // operands.
    //
    // e.g.
    // (X ugt Y) ? X : Y -> (X ole Y) ? Y : X
    if (FCmp->hasOneUse() && FCmpInst::isUnordered(FCmp->getPredicate())) {
      FCmpInst::Predicate InvPred = FCmp->getInversePredicate();
      IRBuilder<>::FastMathFlagGuard FMFG(Builder);
      // FIXME: The FMF should propagate from the select, not the fcmp.
      Builder.setFastMathFlags(FCmp->getFastMathFlags());
      Value *NewCond = Builder.CreateFCmp(InvPred, Cmp0, Cmp1,
                                          FCmp->getName() + ".inv");
      Value *NewSel = Builder.CreateSelect(NewCond, FalseVal, TrueVal);
      return replaceInstUsesWith(SI, NewSel);
    }
  }
}

if (isa<FPMathOperator>(SI)) {
9
←
Assuming 'SI' is not a 'FPMathOperator'→
10
←
Taking false branch→
  // TODO: Try to forward-propagate FMF from select arms to the select.

  // Canonicalize select of FP values where NaN and -0.0 are not valid as
  // minnum/maxnum intrinsics.
  if (SI.hasNoNaNs() && SI.hasNoSignedZeros()) {
    Value *X, *Y;
    if (match(&SI, m_OrdFMax(m_Value(X), m_Value(Y))))
      return replaceInstUsesWith(
          SI, Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, X, Y, &SI));

    if (match(&SI, m_OrdFMin(m_Value(X), m_Value(Y))))
      return replaceInstUsesWith(
          SI, Builder.CreateBinaryIntrinsic(Intrinsic::minnum, X, Y, &SI));
  }
}

// Fold selecting to fabs.
if (Instruction *Fabs10.1
'Fabs' is null
1
'Fabs' is null
 = foldSelectWithFCmpToFabs(SI, *this))
11
←
Taking false branch→
  return Fabs;

// See if we are selecting two values based on a comparison of the two values.
if (ICmpInst *ICI12.1
'ICI' is null
1
'ICI' is null
 = dyn_cast<ICmpInst>(CondVal))
12
←
Assuming 'CondVal' is not a 'CastReturnType'→
13
←
Taking false branch→
  if (Instruction *Result = foldSelectInstWithICmp(SI, ICI))
    return Result;

if (Instruction *Add13.1
'Add' is null
1
'Add' is null
 = foldAddSubSelect(SI, Builder))
14
←
Taking false branch→
  return Add;
if (Instruction *Add14.1
'Add' is null
1
'Add' is null
 = foldOverflowingAddSubSelect(SI, Builder))
15
←
Taking false branch→
  return Add;
if (Instruction *Or15.1
'Or' is null
1
'Or' is null
 = foldSetClearBits(SI, Builder))
16
←
Taking false branch→
  return Or;
if (Instruction *Mul16.1
'Mul' is null
1
'Mul' is null
 = foldSelectZeroOrMul(SI, *this))
17
←
Taking false branch→
  return Mul;

// Turn (select C, (op X, Y), (op X, Z)) -> (op X, (select C, Y, Z))
auto *TI = dyn_cast<Instruction>(TrueVal);
18
←
Assuming 'TrueVal' is not a 'CastReturnType'→
auto *FI = dyn_cast<Instruction>(FalseVal);
19
←
Assuming 'FalseVal' is not a 'CastReturnType'→
if (TI19.1
'TI' is null
1
'TI' is null
 && FI && TI->getOpcode() == FI->getOpcode())
  if (Instruction *IV = foldSelectOpOp(SI, TI, FI))
    return IV;

if (Instruction *I19.2
'I' is null
2
'I' is null
 = foldSelectExtConst(SI))
20
←
Taking false branch→
  return I;

// Fold (select C, (gep Ptr, Idx), Ptr) -> (gep Ptr, (select C, Idx, 0))
// Fold (select C, Ptr, (gep Ptr, Idx)) -> (gep Ptr, (select C, 0, Idx))
auto SelectGepWithBase = [&](GetElementPtrInst *Gep, Value *Base,
                             bool Swap) -> GetElementPtrInst * {
  Value *Ptr = Gep->getPointerOperand();
  if (Gep->getNumOperands() != 2 || Gep->getPointerOperand() != Base ||
      !Gep->hasOneUse())
    return nullptr;
  Value *Idx = Gep->getOperand(1);
  if (isa<VectorType>(CondVal->getType()) && !isa<VectorType>(Idx->getType()))
    return nullptr;
  Type *ElementType = Gep->getResultElementType();
  Value *NewT = Idx;
  Value *NewF = Constant::getNullValue(Idx->getType());
  if (Swap)
    std::swap(NewT, NewF);
  Value *NewSI =
      Builder.CreateSelect(CondVal, NewT, NewF, SI.getName() + ".idx", &SI);
  return GetElementPtrInst::Create(ElementType, Ptr, {NewSI});
};
if (auto *TrueGep21.1
'TrueGep' is null
1
'TrueGep' is null
 = dyn_cast<GetElementPtrInst>(TrueVal))
21
←
Assuming 'TrueVal' is not a 'CastReturnType'→
22
←
Taking false branch→
  if (auto *NewGep = SelectGepWithBase(TrueGep, FalseVal, false))
    return NewGep;
if (auto *FalseGep23.1
'FalseGep' is null
1
'FalseGep' is null
 = dyn_cast<GetElementPtrInst>(FalseVal))
23
←
Assuming 'FalseVal' is not a 'CastReturnType'→
  if (auto *NewGep = SelectGepWithBase(FalseGep, TrueVal, true))
    return NewGep;

// See if we can fold the select into one of our operands.
if (SelType->isIntOrIntVectorTy() || SelType->isFPOrFPVectorTy()) {
24
←
Taking false branch→
  if (Instruction *FoldI = foldSelectIntoOp(SI, TrueVal, FalseVal))
    return FoldI;

  Value *LHS, *RHS;
  Instruction::CastOps CastOp;
  SelectPatternResult SPR = matchSelectPattern(&SI, LHS, RHS, &CastOp);
  auto SPF = SPR.Flavor;
  if (SPF) {
    Value *LHS2, *RHS2;
    if (SelectPatternFlavor SPF2 = matchSelectPattern(LHS, LHS2, RHS2).Flavor)
      if (Instruction *R = foldSPFofSPF(cast<Instruction>(LHS), SPF2, LHS2,
                                        RHS2, SI, SPF, RHS))
        return R;
    if (SelectPatternFlavor SPF2 = matchSelectPattern(RHS, LHS2, RHS2).Flavor)
      if (Instruction *R = foldSPFofSPF(cast<Instruction>(RHS), SPF2, LHS2,
                                        RHS2, SI, SPF, LHS))
        return R;
  }

  if (SelectPatternResult::isMinOrMax(SPF)) {
    // Canonicalize so that
    // - type casts are outside select patterns.
    // - float clamp is transformed to min/max pattern

    bool IsCastNeeded = LHS->getType() != SelType;
    Value *CmpLHS = cast<CmpInst>(CondVal)->getOperand(0);
    Value *CmpRHS = cast<CmpInst>(CondVal)->getOperand(1);
    if (IsCastNeeded ||
        (LHS->getType()->isFPOrFPVectorTy() &&
         ((CmpLHS != LHS && CmpLHS != RHS) ||
          (CmpRHS != LHS && CmpRHS != RHS)))) {
      CmpInst::Predicate MinMaxPred = getMinMaxPred(SPF, SPR.Ordered);

      Value *Cmp;
      if (CmpInst::isIntPredicate(MinMaxPred)) {
        Cmp = Builder.CreateICmp(MinMaxPred, LHS, RHS);
      } else {
        IRBuilder<>::FastMathFlagGuard FMFG(Builder);
        auto FMF =
            cast<FPMathOperator>(SI.getCondition())->getFastMathFlags();
        Builder.setFastMathFlags(FMF);
        Cmp = Builder.CreateFCmp(MinMaxPred, LHS, RHS);
      }

      Value *NewSI = Builder.CreateSelect(Cmp, LHS, RHS, SI.getName(), &SI);
      if (!IsCastNeeded)
        return replaceInstUsesWith(SI, NewSI);

      Value *NewCast = Builder.CreateCast(CastOp, NewSI, SelType);
      return replaceInstUsesWith(SI, NewCast);
    }
  }
}

// See if we can fold the select into a phi node if the condition is a select.
if (auto *PN25.1
'PN' is null
1
'PN' is null
 = dyn_cast<PHINode>(SI.getCondition()))
25
←
Assuming the object is not a 'CastReturnType'→
26
←
Taking false branch→
  // The true/false values have to be live in the PHI predecessor's blocks.
  if (canSelectOperandBeMappingIntoPredBlock(TrueVal, SI) &&
      canSelectOperandBeMappingIntoPredBlock(FalseVal, SI))
    if (Instruction *NV = foldOpIntoPhi(SI, PN))
      return NV;

if (SelectInst *TrueSI27.1
'TrueSI' is null
1
'TrueSI' is null
 = dyn_cast<SelectInst>(TrueVal)) {
27
←
Assuming 'TrueVal' is not a 'CastReturnType'→
28
←
Taking false branch→
  if (TrueSI->getCondition()->getType() == CondVal->getType()) {
    // select(C, select(C, a, b), c) -> select(C, a, c)
    if (TrueSI->getCondition() == CondVal) {
      if (SI.getTrueValue() == TrueSI->getTrueValue())
        return nullptr;
      return replaceOperand(SI, 1, TrueSI->getTrueValue());
    }
    // select(C0, select(C1, a, b), b) -> select(C0&C1, a, b)
    // We choose this as normal form to enable folding on the And and
    // shortening paths for the values (this helps getUnderlyingObjects() for
    // example).
    if (TrueSI->getFalseValue() == FalseVal && TrueSI->hasOneUse()) {
      Value *And = Builder.CreateLogicalAnd(CondVal, TrueSI->getCondition());
      replaceOperand(SI, 0, And);
      replaceOperand(SI, 1, TrueSI->getTrueValue());
      return &SI;
    }
  }
}
if (SelectInst *FalseSI29.1
'FalseSI' is null
1
'FalseSI' is null
 = dyn_cast<SelectInst>(FalseVal)) {
29
←
Assuming 'FalseVal' is not a 'CastReturnType'→
30
←
Taking false branch→
  if (FalseSI->getCondition()->getType() == CondVal->getType()) {
    // select(C, a, select(C, b, c)) -> select(C, a, c)
    if (FalseSI->getCondition() == CondVal) {
      if (SI.getFalseValue() == FalseSI->getFalseValue())
        return nullptr;
      return replaceOperand(SI, 2, FalseSI->getFalseValue());
    }
    // select(C0, a, select(C1, a, b)) -> select(C0|C1, a, b)
    if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) {
      Value *Or = Builder.CreateLogicalOr(CondVal, FalseSI->getCondition());
      replaceOperand(SI, 0, Or);
      replaceOperand(SI, 2, FalseSI->getFalseValue());
      return &SI;
    }
  }
}

// Try to simplify a binop sandwiched between 2 selects with the same
// condition. This is not valid for div/rem because the select might be
// preventing a division-by-zero.
// TODO: A div/rem restriction is conservative; use something like
//       isSafeToSpeculativelyExecute().
// select(C, binop(select(C, X, Y), W), Z) -> select(C, binop(X, W), Z)
BinaryOperator *TrueBO;
if (match(TrueVal, m_OneUse(m_BinOp(TrueBO))) && !TrueBO->isIntDivRem()) {
31
←
Taking false branch→
  if (auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->getOperand(0))) {
    if (TrueBOSI->getCondition() == CondVal) {
      replaceOperand(*TrueBO, 0, TrueBOSI->getTrueValue());
      Worklist.push(TrueBO);
      return &SI;
    }
  }
  if (auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->getOperand(1))) {
    if (TrueBOSI->getCondition() == CondVal) {
      replaceOperand(*TrueBO, 1, TrueBOSI->getTrueValue());
      Worklist.push(TrueBO);
      return &SI;
    }
  }
}

// select(C, Z, binop(select(C, X, Y), W)) -> select(C, Z, binop(Y, W))
BinaryOperator *FalseBO;
if (match(FalseVal, m_OneUse(m_BinOp(FalseBO))) && !FalseBO->isIntDivRem()) {
32
←
Taking false branch→
  if (auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->getOperand(0))) {
    if (FalseBOSI->getCondition() == CondVal) {
      replaceOperand(*FalseBO, 0, FalseBOSI->getFalseValue());
      Worklist.push(FalseBO);
      return &SI;
    }
  }
  if (auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->getOperand(1))) {
    if (FalseBOSI->getCondition() == CondVal) {
      replaceOperand(*FalseBO, 1, FalseBOSI->getFalseValue());
      Worklist.push(FalseBO);
      return &SI;
    }
  }
}

Value *NotCond;
if (match(CondVal, m_Not(m_Value(NotCond))) &&
33
←
Taking false branch→
    !InstCombiner::shouldAvoidAbsorbingNotIntoSelect(SI)) {
  replaceOperand(SI, 0, NotCond);
  SI.swapValues();
  SI.swapProfMetadata();
  return &SI;
}

if (Instruction *I33.1
'I' is null
1
'I' is null
 = foldVectorSelect(SI))
  return I;

// If we can compute the condition, there's no need for a select.
// Like the above fold, we are attempting to reduce compile-time cost by
// putting this fold here with limitations rather than in InstSimplify.
// The motivation for this call into value tracking is to take advantage of
// the assumption cache, so make sure that is populated.
if (!CondVal->getType()->isVectorTy() && !AC.assumptions().empty()) {
34
←
Assuming the condition is false→
35
←
Taking false branch→
  KnownBits Known(1);
  computeKnownBits(CondVal, Known, 0, &SI);
  if (Known.One.isOne())
    return replaceInstUsesWith(SI, TrueVal);
  if (Known.Zero.isOne())
    return replaceInstUsesWith(SI, FalseVal);
}

if (Instruction *BitCastSel35.1
'BitCastSel' is null
1
'BitCastSel' is null
 = foldSelectCmpBitcasts(SI, Builder))
36
←
Taking false branch→
  return BitCastSel;

// Simplify selects that test the returned flag of cmpxchg instructions.
if (Value *V36.1
'V' is null
1
'V' is null
 = foldSelectCmpXchg(SI))
37
←
Taking false branch→
  return replaceInstUsesWith(SI, V);

if (Instruction *Select37.1
'Select' is null
1
'Select' is null
 = foldSelectBinOpIdentity(SI, TLI, *this))
38
←
Taking false branch→
  return Select;

if (Instruction *Funnel38.1
'Funnel' is null
1
'Funnel' is null
 = foldSelectFunnelShift(SI, Builder))
39
←
Taking false branch→
  return Funnel;

if (Instruction *Copysign39.1
'Copysign' is null
1
'Copysign' is null
 = foldSelectToCopysign(SI, Builder))
40
←
Taking false branch→
  return Copysign;

if (Instruction *PN40.1
'PN' is null
1
'PN' is null
 = foldSelectToPhi(SI, DT, Builder))
41
←
Taking false branch→
  return replaceInstUsesWith(SI, PN);

if (Value *Fr41.1
'Fr' is null
1
'Fr' is null
 = foldSelectWithFrozenICmp(SI, Builder))
42
←
Taking false branch→
  return replaceInstUsesWith(SI, Fr);

if (Value *V42.1
'V' is null
1
'V' is null
 = foldRoundUpIntegerWithPow2Alignment(SI, Builder))
  return replaceInstUsesWith(SI, V);

// select(mask, mload(,,mask,0), 0) -> mload(,,mask,0)
// Load inst is intentionally not checked for hasOneUse()
if (match(FalseVal, m_Zero()) &&
43
←
Taking false branch→
    (match(TrueVal, m_MaskedLoad(m_Value(), m_Value(), m_Specific(CondVal),
                                 m_CombineOr(m_Undef(), m_Zero()))) ||
     match(TrueVal, m_MaskedGather(m_Value(), m_Value(), m_Specific(CondVal),
                                   m_CombineOr(m_Undef(), m_Zero()))))) {
  auto *MaskedInst = cast<IntrinsicInst>(TrueVal);
  if (isa<UndefValue>(MaskedInst->getArgOperand(3)))
    MaskedInst->setArgOperand(3, FalseVal /* Zero */);
  return replaceInstUsesWith(SI, MaskedInst);
}

Value *Mask;
if (match(TrueVal, m_Zero()) &&
44
←
Taking false branch→
    (match(FalseVal, m_MaskedLoad(m_Value(), m_Value(), m_Value(Mask),
                                  m_CombineOr(m_Undef(), m_Zero()))) ||
     match(FalseVal, m_MaskedGather(m_Value(), m_Value(), m_Value(Mask),
                                    m_CombineOr(m_Undef(), m_Zero())))) &&
    (CondVal->getType() == Mask->getType())) {
  // We can remove the select by ensuring the load zeros all lanes the
  // select would have.  We determine this by proving there is no overlap
  // between the load and select masks.
  // (i.e (load_mask & select_mask) == 0 == no overlap)
  bool CanMergeSelectIntoLoad = false;
  if (Value *V = simplifyAndInst(CondVal, Mask, SQ.getWithInstruction(&SI)))
    CanMergeSelectIntoLoad = match(V, m_Zero());

  if (CanMergeSelectIntoLoad) {
    auto *MaskedInst = cast<IntrinsicInst>(FalseVal);
    if (isa<UndefValue>(MaskedInst->getArgOperand(3)))
      MaskedInst->setArgOperand(3, TrueVal /* Zero */);
    return replaceInstUsesWith(SI, MaskedInst);
  }
}

if (Instruction *I = foldNestedSelects(SI, Builder))
45
←
Calling 'foldNestedSelects'→
  return I;

// Match logical variants of the pattern,
// and transform them iff that gets rid of inversions.
//   (~x) | y  -->  ~(x & (~y))
//   (~x) & y  -->  ~(x | (~y))
if (sinkNotIntoOtherHandOfLogicalOp(SI))
  return &SI;

if (Instruction *I = foldBitCeil(SI, Builder))
  return I;

return nullptr;
3725}

←

/build/source/llvm/include/llvm/IR/PatternMatch.h

1//===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the LLVM IR. The power of these routines is
11// that it allows you to write concise patterns that are expressive and easy to
12// understand. The other major advantage of this is that it allows you to
13// trivially capture/bind elements in the pattern to variables. For example,
14// you can do something like this:
15//
16//  Value *Exp = ...
17//  Value *X, *Y;  ConstantInt *C1, *C2;      // (X & C1) | (Y & C2)
18//  if (match(Exp, m_Or(m_And(m_Value(X), m_ConstantInt(C1)),
19//                      m_And(m_Value(Y), m_ConstantInt(C2))))) {
20//    ... Pattern is matched and variables are bound ...
21//  }
22//
23// This is primarily useful to things like the instruction combiner, but can
24// also be useful for static analysis tools or code generators.
25//
26//===----------------------------------------------------------------------===//
27 
28#ifndef LLVM_IR_PATTERNMATCH_H
29#define LLVM_IR_PATTERNMATCH_H
30 
31#include "llvm/ADT/APFloat.h"
32#include "llvm/ADT/APInt.h"
33#include "llvm/IR/Constant.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DataLayout.h"
36#include "llvm/IR/InstrTypes.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/IntrinsicInst.h"
40#include "llvm/IR/Intrinsics.h"
41#include "llvm/IR/Operator.h"
42#include "llvm/IR/Value.h"
43#include "llvm/Support/Casting.h"
44#include <cstdint>
45 
46namespace llvm {
47namespace PatternMatch {
48 
49template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
50  return const_cast<Pattern &>(P).match(V);
58
←
Returning without writing to 'P.Op1.VR'→
73
←
Passing null pointer value via 1st parameter 'V'→
74
←
Calling 'BinaryOp_match::match'→
51}
52 
53template <typename Pattern> bool match(ArrayRef<int> Mask, const Pattern &P) {
54  return const_cast<Pattern &>(P).match(Mask);
55}
56 
57template <typename SubPattern_t> struct OneUse_match {
58  SubPattern_t SubPattern;
59 
60  OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
61 
62  template <typename OpTy> bool match(OpTy *V) {
63    return V->hasOneUse() && SubPattern.match(V);
64  }
65};
66 
67template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
68  return SubPattern;
69}
70 
71template <typename Class> struct class_match {
72  template <typename ITy> bool match(ITy *V) { return isa<Class>(V); }
73};
74 
75/// Match an arbitrary value and ignore it.
76inline class_match<Value> m_Value() { return class_match<Value>(); }
77 
78/// Match an arbitrary unary operation and ignore it.
79inline class_match<UnaryOperator> m_UnOp() {
80  return class_match<UnaryOperator>();
81}
82 
83/// Match an arbitrary binary operation and ignore it.
84inline class_match<BinaryOperator> m_BinOp() {
85  return class_match<BinaryOperator>();
86}
87 
88/// Matches any compare instruction and ignore it.
89inline class_match<CmpInst> m_Cmp() { return class_match<CmpInst>(); }
90 
91struct undef_match {
92  static bool check(const Value *V) {
93    if (isa<UndefValue>(V))
94      return true;
95 
96    const auto *CA = dyn_cast<ConstantAggregate>(V);
97    if (!CA)
98      return false;
99 
100    SmallPtrSet<const ConstantAggregate *, 8> Seen;
101    SmallVector<const ConstantAggregate *, 8> Worklist;
102 
103    // Either UndefValue, PoisonValue, or an aggregate that only contains
104    // these is accepted by matcher.
105    // CheckValue returns false if CA cannot satisfy this constraint.
106    auto CheckValue = [&](const ConstantAggregate *CA) {
107      for (const Value *Op : CA->operand_values()) {
108        if (isa<UndefValue>(Op))
109          continue;
110 
111        const auto *CA = dyn_cast<ConstantAggregate>(Op);
112        if (!CA)
113          return false;
114        if (Seen.insert(CA).second)
115          Worklist.emplace_back(CA);
116      }
117 
118      return true;
119    };
120 
121    if (!CheckValue(CA))
122      return false;
123 
124    while (!Worklist.empty()) {
125      if (!CheckValue(Worklist.pop_back_val()))
126        return false;
127    }
128    return true;
129  }
130  template <typename ITy> bool match(ITy *V) { return check(V); }
131};
132 
133/// Match an arbitrary undef constant. This matches poison as well.
134/// If this is an aggregate and contains a non-aggregate element that is
135/// neither undef nor poison, the aggregate is not matched.
136inline auto m_Undef() { return undef_match(); }
137 
138/// Match an arbitrary poison constant.
139inline class_match<PoisonValue> m_Poison() {
140  return class_match<PoisonValue>();
141}
142 
143/// Match an arbitrary Constant and ignore it.
144inline class_match<Constant> m_Constant() { return class_match<Constant>(); }
145 
146/// Match an arbitrary ConstantInt and ignore it.
147inline class_match<ConstantInt> m_ConstantInt() {
148  return class_match<ConstantInt>();
149}
150 
151/// Match an arbitrary ConstantFP and ignore it.
152inline class_match<ConstantFP> m_ConstantFP() {
153  return class_match<ConstantFP>();
154}
155 
156struct constantexpr_match {
157  template <typename ITy> bool match(ITy *V) {
158    auto *C = dyn_cast<Constant>(V);
159    return C && (isa<ConstantExpr>(C) || C->containsConstantExpression());
160  }
161};
162 
163/// Match a constant expression or a constant that contains a constant
164/// expression.
165inline constantexpr_match m_ConstantExpr() { return constantexpr_match(); }
166 
167/// Match an arbitrary basic block value and ignore it.
168inline class_match<BasicBlock> m_BasicBlock() {
169  return class_match<BasicBlock>();
170}
171 
172/// Inverting matcher
173template <typename Ty> struct match_unless {
174  Ty M;
175 
176  match_unless(const Ty &Matcher) : M(Matcher) {}
177 
178  template <typename ITy> bool match(ITy *V) { return !M.match(V); }
179};
180 
181/// Match if the inner matcher does *NOT* match.
182template <typename Ty> inline match_unless<Ty> m_Unless(const Ty &M) {
183  return match_unless<Ty>(M);
184}
185 
186/// Matching combinators
187template <typename LTy, typename RTy> struct match_combine_or {
188  LTy L;
189  RTy R;
190 
191  match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
192 
193  template <typename ITy> bool match(ITy *V) {
194    if (L.match(V))
195      return true;
196    if (R.match(V))
197      return true;
198    return false;
199  }
200};
201 
202template <typename LTy, typename RTy> struct match_combine_and {
203  LTy L;
204  RTy R;
205 
206  match_combine_and(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
207 
208  template <typename ITy> bool match(ITy *V) {
209    if (L.match(V))
210      if (R.match(V))
211        return true;
212    return false;
213  }
214};
215 
216/// Combine two pattern matchers matching L || R
217template <typename LTy, typename RTy>
218inline match_combine_or<LTy, RTy> m_CombineOr(const LTy &L, const RTy &R) {
219  return match_combine_or<LTy, RTy>(L, R);
220}
221 
222/// Combine two pattern matchers matching L && R
223template <typename LTy, typename RTy>
224inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {
225  return match_combine_and<LTy, RTy>(L, R);
226}
227 
228struct apint_match {
229  const APInt *&Res;
230  bool AllowUndef;
231 
232  apint_match(const APInt *&Res, bool AllowUndef)
233      : Res(Res), AllowUndef(AllowUndef) {}
234 
235  template <typename ITy> bool match(ITy *V) {
236    if (auto *CI = dyn_cast<ConstantInt>(V)) {
237      Res = &CI->getValue();
238      return true;
239    }
240    if (V->getType()->isVectorTy())
241      if (const auto *C = dyn_cast<Constant>(V))
242        if (auto *CI =
243                dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndef))) {
244          Res = &CI->getValue();
245          return true;
246        }
247    return false;
248  }
249};
250// Either constexpr if or renaming ConstantFP::getValueAPF to
251// ConstantFP::getValue is needed to do it via single template
252// function for both apint/apfloat.
253struct apfloat_match {
254  const APFloat *&Res;
255  bool AllowUndef;
256 
257  apfloat_match(const APFloat *&Res, bool AllowUndef)
258      : Res(Res), AllowUndef(AllowUndef) {}
259 
260  template <typename ITy> bool match(ITy *V) {
261    if (auto *CI = dyn_cast<ConstantFP>(V)) {
262      Res = &CI->getValueAPF();
263      return true;
264    }
265    if (V->getType()->isVectorTy())
266      if (const auto *C = dyn_cast<Constant>(V))
267        if (auto *CI =
268                dyn_cast_or_null<ConstantFP>(C->getSplatValue(AllowUndef))) {
269          Res = &CI->getValueAPF();
270          return true;
271        }
272    return false;
273  }
274};
275 
276/// Match a ConstantInt or splatted ConstantVector, binding the
277/// specified pointer to the contained APInt.
278inline apint_match m_APInt(const APInt *&Res) {
279  // Forbid undefs by default to maintain previous behavior.
280  return apint_match(Res, /* AllowUndef */ false);
281}
282 
283/// Match APInt while allowing undefs in splat vector constants.
284inline apint_match m_APIntAllowUndef(const APInt *&Res) {
285  return apint_match(Res, /* AllowUndef */ true);
286}
287 
288/// Match APInt while forbidding undefs in splat vector constants.
289inline apint_match m_APIntForbidUndef(const APInt *&Res) {
290  return apint_match(Res, /* AllowUndef */ false);
291}
292 
293/// Match a ConstantFP or splatted ConstantVector, binding the
294/// specified pointer to the contained APFloat.
295inline apfloat_match m_APFloat(const APFloat *&Res) {
296  // Forbid undefs by default to maintain previous behavior.
297  return apfloat_match(Res, /* AllowUndef */ false);
298}
299 
300/// Match APFloat while allowing undefs in splat vector constants.
301inline apfloat_match m_APFloatAllowUndef(const APFloat *&Res) {
302  return apfloat_match(Res, /* AllowUndef */ true);
303}
304 
305/// Match APFloat while forbidding undefs in splat vector constants.
306inline apfloat_match m_APFloatForbidUndef(const APFloat *&Res) {
307  return apfloat_match(Res, /* AllowUndef */ false);
308}
309 
310template <int64_t Val> struct constantint_match {
311  template <typename ITy> bool match(ITy *V) {
312    if (const auto *CI = dyn_cast<ConstantInt>(V)) {
313      const APInt &CIV = CI->getValue();
314      if (Val >= 0)
315        return CIV == static_cast<uint64_t>(Val);
316      // If Val is negative, and CI is shorter than it, truncate to the right
317      // number of bits.  If it is larger, then we have to sign extend.  Just
318      // compare their negated values.
319      return -CIV == -Val;
320    }
321    return false;
322  }
323};
324 
325/// Match a ConstantInt with a specific value.
326template <int64_t Val> inline constantint_match<Val> m_ConstantInt() {
327  return constantint_match<Val>();
328}
329 
330/// This helper class is used to match constant scalars, vector splats,
331/// and fixed width vectors that satisfy a specified predicate.
332/// For fixed width vector constants, undefined elements are ignored.
333template <typename Predicate, typename ConstantVal>
334struct cstval_pred_ty : public Predicate {
335  template <typename ITy> bool match(ITy *V) {
336    if (const auto *CV = dyn_cast<ConstantVal>(V))
337      return this->isValue(CV->getValue());
338    if (const auto *VTy = dyn_cast<VectorType>(V->getType())) {
339      if (const auto *C = dyn_cast<Constant>(V)) {
340        if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue()))
341          return this->isValue(CV->getValue());
342 
343        // Number of elements of a scalable vector unknown at compile time
344        auto *FVTy = dyn_cast<FixedVectorType>(VTy);
345        if (!FVTy)
346          return false;
347 
348        // Non-splat vector constant: check each element for a match.
349        unsigned NumElts = FVTy->getNumElements();
350        assert(NumElts != 0 && "Constant vector with no elements?")(static_cast <bool> (NumElts != 0 && "Constant vector with no elements?"
) ? void (0) : __assert_fail ("NumElts != 0 && \"Constant vector with no elements?\""
, "llvm/include/llvm/IR/PatternMatch.h", 350, __extension__ __PRETTY_FUNCTION__
));
351        bool HasNonUndefElements = false;
352        for (unsigned i = 0; i != NumElts; ++i) {
353          Constant *Elt = C->getAggregateElement(i);
354          if (!Elt)
355            return false;
356          if (isa<UndefValue>(Elt))
357            continue;
358          auto *CV = dyn_cast<ConstantVal>(Elt);
359          if (!CV || !this->isValue(CV->getValue()))
360            return false;
361          HasNonUndefElements = true;
362        }
363        return HasNonUndefElements;
364      }
365    }
366    return false;
367  }
368};
369 
370/// specialization of cstval_pred_ty for ConstantInt
371template <typename Predicate>
372using cst_pred_ty = cstval_pred_ty<Predicate, ConstantInt>;
373 
374/// specialization of cstval_pred_ty for ConstantFP
375template <typename Predicate>
376using cstfp_pred_ty = cstval_pred_ty<Predicate, ConstantFP>;
377 
378/// This helper class is used to match scalar and vector constants that
379/// satisfy a specified predicate, and bind them to an APInt.
380template <typename Predicate> struct api_pred_ty : public Predicate {
381  const APInt *&Res;
382 
383  api_pred_ty(const APInt *&R) : Res(R) {}
384 
385  template <typename ITy> bool match(ITy *V) {
386    if (const auto *CI = dyn_cast<ConstantInt>(V))
387      if (this->isValue(CI->getValue())) {
388        Res = &CI->getValue();
389        return true;
390      }
391    if (V->getType()->isVectorTy())
392      if (const auto *C = dyn_cast<Constant>(V))
393        if (auto *CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue()))
394          if (this->isValue(CI->getValue())) {
395            Res = &CI->getValue();
396            return true;
397          }
398 
399    return false;
400  }
401};
402 
403/// This helper class is used to match scalar and vector constants that
404/// satisfy a specified predicate, and bind them to an APFloat.
405/// Undefs are allowed in splat vector constants.
406template <typename Predicate> struct apf_pred_ty : public Predicate {
407  const APFloat *&Res;
408 
409  apf_pred_ty(const APFloat *&R) : Res(R) {}
410 
411  template <typename ITy> bool match(ITy *V) {
412    if (const auto *CI = dyn_cast<ConstantFP>(V))
413      if (this->isValue(CI->getValue())) {
414        Res = &CI->getValue();
415        return true;
416      }
417    if (V->getType()->isVectorTy())
418      if (const auto *C = dyn_cast<Constant>(V))
419        if (auto *CI = dyn_cast_or_null<ConstantFP>(
420                C->getSplatValue(/* AllowUndef */ true)))
421          if (this->isValue(CI->getValue())) {
422            Res = &CI->getValue();
423            return true;
424          }
425 
426    return false;
427  }
428};
429 
430///////////////////////////////////////////////////////////////////////////////
431//
432// Encapsulate constant value queries for use in templated predicate matchers.
433// This allows checking if constants match using compound predicates and works
434// with vector constants, possibly with relaxed constraints. For example, ignore
435// undef values.
436//
437///////////////////////////////////////////////////////////////////////////////
438 
439struct is_any_apint {
440  bool isValue(const APInt &C) { return true; }
441};
442/// Match an integer or vector with any integral constant.
443/// For vectors, this includes constants with undefined elements.
444inline cst_pred_ty<is_any_apint> m_AnyIntegralConstant() {
445  return cst_pred_ty<is_any_apint>();
446}
447 
448struct is_all_ones {
449  bool isValue(const APInt &C) { return C.isAllOnes(); }
450};
451/// Match an integer or vector with all bits set.
452/// For vectors, this includes constants with undefined elements.
453inline cst_pred_ty<is_all_ones> m_AllOnes() {
454  return cst_pred_ty<is_all_ones>();
455}
456 
457struct is_maxsignedvalue {
458  bool isValue(const APInt &C) { return C.isMaxSignedValue(); }
459};
460/// Match an integer or vector with values having all bits except for the high
461/// bit set (0x7f...).
462/// For vectors, this includes constants with undefined elements.
463inline cst_pred_ty<is_maxsignedvalue> m_MaxSignedValue() {
464  return cst_pred_ty<is_maxsignedvalue>();
465}
466inline api_pred_ty<is_maxsignedvalue> m_MaxSignedValue(const APInt *&V) {
467  return V;
468}
469 
470struct is_negative {
471  bool isValue(const APInt &C) { return C.isNegative(); }
472};
473/// Match an integer or vector of negative values.
474/// For vectors, this includes constants with undefined elements.
475inline cst_pred_ty<is_negative> m_Negative() {
476  return cst_pred_ty<is_negative>();
477}
478inline api_pred_ty<is_negative> m_Negative(const APInt *&V) { return V; }
479 
480struct is_nonnegative {
481  bool isValue(const APInt &C) { return C.isNonNegative(); }
482};
483/// Match an integer or vector of non-negative values.
484/// For vectors, this includes constants with undefined elements.
485inline cst_pred_ty<is_nonnegative> m_NonNegative() {
486  return cst_pred_ty<is_nonnegative>();
487}
488inline api_pred_ty<is_nonnegative> m_NonNegative(const APInt *&V) { return V; }
489 
490struct is_strictlypositive {
491  bool isValue(const APInt &C) { return C.isStrictlyPositive(); }
492};
493/// Match an integer or vector of strictly positive values.
494/// For vectors, this includes constants with undefined elements.
495inline cst_pred_ty<is_strictlypositive> m_StrictlyPositive() {
496  return cst_pred_ty<is_strictlypositive>();
497}
498inline api_pred_ty<is_strictlypositive> m_StrictlyPositive(const APInt *&V) {
499  return V;
500}
501 
502struct is_nonpositive {
503  bool isValue(const APInt &C) { return C.isNonPositive(); }
504};
505/// Match an integer or vector of non-positive values.
506/// For vectors, this includes constants with undefined elements.
507inline cst_pred_ty<is_nonpositive> m_NonPositive() {
508  return cst_pred_ty<is_nonpositive>();
509}
510inline api_pred_ty<is_nonpositive> m_NonPositive(const APInt *&V) { return V; }
511 
512struct is_one {
513  bool isValue(const APInt &C) { return C.isOne(); }
514};
515/// Match an integer 1 or a vector with all elements equal to 1.
516/// For vectors, this includes constants with undefined elements.
517inline cst_pred_ty<is_one> m_One() { return cst_pred_ty<is_one>(); }
518 
519struct is_zero_int {
520  bool isValue(const APInt &C) { return C.isZero(); }
521};
522/// Match an integer 0 or a vector with all elements equal to 0.
523/// For vectors, this includes constants with undefined elements.
524inline cst_pred_ty<is_zero_int> m_ZeroInt() {
525  return cst_pred_ty<is_zero_int>();
526}
527 
528struct is_zero {
529  template <typename ITy> bool match(ITy *V) {
530    auto *C = dyn_cast<Constant>(V);
531    // FIXME: this should be able to do something for scalable vectors
532    return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C));
533  }
534};
535/// Match any null constant or a vector with all elements equal to 0.
536/// For vectors, this includes constants with undefined elements.
537inline is_zero m_Zero() { return is_zero(); }
538 
539struct is_power2 {
540  bool isValue(const APInt &C) { return C.isPowerOf2(); }
541};
542/// Match an integer or vector power-of-2.
543/// For vectors, this includes constants with undefined elements.
544inline cst_pred_ty<is_power2> m_Power2() { return cst_pred_ty<is_power2>(); }
545inline api_pred_ty<is_power2> m_Power2(const APInt *&V) { return V; }
546 
547struct is_negated_power2 {
548  bool isValue(const APInt &C) { return C.isNegatedPowerOf2(); }
549};
550/// Match a integer or vector negated power-of-2.
551/// For vectors, this includes constants with undefined elements.
552inline cst_pred_ty<is_negated_power2> m_NegatedPower2() {
553  return cst_pred_ty<is_negated_power2>();
554}
555inline api_pred_ty<is_negated_power2> m_NegatedPower2(const APInt *&V) {
556  return V;
557}
558 
559struct is_power2_or_zero {
560  bool isValue(const APInt &C) { return !C || C.isPowerOf2(); }
561};
562/// Match an integer or vector of 0 or power-of-2 values.
563/// For vectors, this includes constants with undefined elements.
564inline cst_pred_ty<is_power2_or_zero> m_Power2OrZero() {
565  return cst_pred_ty<is_power2_or_zero>();
566}
567inline api_pred_ty<is_power2_or_zero> m_Power2OrZero(const APInt *&V) {
568  return V;
569}
570 
571struct is_sign_mask {
572  bool isValue(const APInt &C) { return C.isSignMask(); }
573};
574/// Match an integer or vector with only the sign bit(s) set.
575/// For vectors, this includes constants with undefined elements.
576inline cst_pred_ty<is_sign_mask> m_SignMask() {
577  return cst_pred_ty<is_sign_mask>();
578}
579 
580struct is_lowbit_mask {
581  bool isValue(const APInt &C) { return C.isMask(); }
582};
583/// Match an integer or vector with only the low bit(s) set.
584/// For vectors, this includes constants with undefined elements.
585inline cst_pred_ty<is_lowbit_mask> m_LowBitMask() {
586  return cst_pred_ty<is_lowbit_mask>();
587}
588inline api_pred_ty<is_lowbit_mask> m_LowBitMask(const APInt *&V) { return V; }
589 
590struct icmp_pred_with_threshold {
591  ICmpInst::Predicate Pred;
592  const APInt *Thr;
593  bool isValue(const APInt &C) { return ICmpInst::compare(C, *Thr, Pred); }
594};
595/// Match an integer or vector with every element comparing 'pred' (eg/ne/...)
596/// to Threshold. For vectors, this includes constants with undefined elements.
597inline cst_pred_ty<icmp_pred_with_threshold>
598m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) {
599  cst_pred_ty<icmp_pred_with_threshold> P;
600  P.Pred = Predicate;
601  P.Thr = &Threshold;
602  return P;
603}
604 
605struct is_nan {
606  bool isValue(const APFloat &C) { return C.isNaN(); }
607};
608/// Match an arbitrary NaN constant. This includes quiet and signalling nans.
609/// For vectors, this includes constants with undefined elements.
610inline cstfp_pred_ty<is_nan> m_NaN() { return cstfp_pred_ty<is_nan>(); }
611 
612struct is_nonnan {
613  bool isValue(const APFloat &C) { return !C.isNaN(); }
614};
615/// Match a non-NaN FP constant.
616/// For vectors, this includes constants with undefined elements.
617inline cstfp_pred_ty<is_nonnan> m_NonNaN() {
618  return cstfp_pred_ty<is_nonnan>();
619}
620 
621struct is_inf {
622  bool isValue(const APFloat &C) { return C.isInfinity(); }
623};
624/// Match a positive or negative infinity FP constant.
625/// For vectors, this includes constants with undefined elements.
626inline cstfp_pred_ty<is_inf> m_Inf() { return cstfp_pred_ty<is_inf>(); }
627 
628struct is_noninf {
629  bool isValue(const APFloat &C) { return !C.isInfinity(); }
630};
631/// Match a non-infinity FP constant, i.e. finite or NaN.
632/// For vectors, this includes constants with undefined elements.
633inline cstfp_pred_ty<is_noninf> m_NonInf() {
634  return cstfp_pred_ty<is_noninf>();
635}
636 
637struct is_finite {
638  bool isValue(const APFloat &C) { return C.isFinite(); }
639};
640/// Match a finite FP constant, i.e. not infinity or NaN.
641/// For vectors, this includes constants with undefined elements.
642inline cstfp_pred_ty<is_finite> m_Finite() {
643  return cstfp_pred_ty<is_finite>();
644}
645inline apf_pred_ty<is_finite> m_Finite(const APFloat *&V) { return V; }
646 
647struct is_finitenonzero {
648  bool isValue(const APFloat &C) { return C.isFiniteNonZero(); }
649};
650/// Match a finite non-zero FP constant.
651/// For vectors, this includes constants with undefined elements.
652inline cstfp_pred_ty<is_finitenonzero> m_FiniteNonZero() {
653  return cstfp_pred_ty<is_finitenonzero>();
654}
655inline apf_pred_ty<is_finitenonzero> m_FiniteNonZero(const APFloat *&V) {
656  return V;
657}
658 
659struct is_any_zero_fp {
660  bool isValue(const APFloat &C) { return C.isZero(); }
661};
662/// Match a floating-point negative zero or positive zero.
663/// For vectors, this includes constants with undefined elements.
664inline cstfp_pred_ty<is_any_zero_fp> m_AnyZeroFP() {
665  return cstfp_pred_ty<is_any_zero_fp>();
666}
667 
668struct is_pos_zero_fp {
669  bool isValue(const APFloat &C) { return C.isPosZero(); }
670};
671/// Match a floating-point positive zero.
672/// For vectors, this includes constants with undefined elements.
673inline cstfp_pred_ty<is_pos_zero_fp> m_PosZeroFP() {
674  return cstfp_pred_ty<is_pos_zero_fp>();
675}
676 
677struct is_neg_zero_fp {
678  bool isValue(const APFloat &C) { return C.isNegZero(); }
679};
680/// Match a floating-point negative zero.
681/// For vectors, this includes constants with undefined elements.
682inline cstfp_pred_ty<is_neg_zero_fp> m_NegZeroFP() {
683  return cstfp_pred_ty<is_neg_zero_fp>();
684}
685 
686struct is_non_zero_fp {
687  bool isValue(const APFloat &C) { return C.isNonZero(); }
688};
689/// Match a floating-point non-zero.
690/// For vectors, this includes constants with undefined elements.
691inline cstfp_pred_ty<is_non_zero_fp> m_NonZeroFP() {
692  return cstfp_pred_ty<is_non_zero_fp>();
693}
694 
695///////////////////////////////////////////////////////////////////////////////
696 
697template <typename Class> struct bind_ty {
698  Class *&VR;
699 
700  bind_ty(Class *&V) : VR(V) {}
701 
702  template <typename ITy> bool match(ITy *V) {
703    if (auto *CV = dyn_cast<Class>(V)) {
704      VR = CV;
705      return true;
706    }
707    return false;
708  }
709};
710 
711/// Match a value, capturing it if we match.
712inline bind_ty<Value> m_Value(Value *&V) { return V; }
50
←
Calling constructor for 'bind_ty<llvm::Value>'→
51
←
Returning from constructor for 'bind_ty<llvm::Value>'→
52
←
Returning without writing to 'V'→
61
←
Calling constructor for 'bind_ty<llvm::Value>'→
62
←
Returning from constructor for 'bind_ty<llvm::Value>'→
63
←
Returning without writing to 'V'→
713inline bind_ty<const Value> m_Value(const Value *&V) { return V; }
714 
715/// Match an instruction, capturing it if we match.
716inline bind_ty<Instruction> m_Instruction(Instruction *&I) { return I; }
717/// Match a unary operator, capturing it if we match.
718inline bind_ty<UnaryOperator> m_UnOp(UnaryOperator *&I) { return I; }
719/// Match a binary operator, capturing it if we match.
720inline bind_ty<BinaryOperator> m_BinOp(BinaryOperator *&I) { return I; }
721/// Match a with overflow intrinsic, capturing it if we match.
722inline bind_ty<WithOverflowInst> m_WithOverflowInst(WithOverflowInst *&I) {
723  return I;
724}
725inline bind_ty<const WithOverflowInst>
726m_WithOverflowInst(const WithOverflowInst *&I) {
727  return I;
728}
729 
730/// Match a Constant, capturing the value if we match.
731inline bind_ty<Constant> m_Constant(Constant *&C) { return C; }
732 
733/// Match a ConstantInt, capturing the value if we match.
734inline bind_ty<ConstantInt> m_ConstantInt(ConstantInt *&CI) { return CI; }
735 
736/// Match a ConstantFP, capturing the value if we match.
737inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; }
738 
739/// Match a ConstantExpr, capturing the value if we match.
740inline bind_ty<ConstantExpr> m_ConstantExpr(ConstantExpr *&C) { return C; }
741 
742/// Match a basic block value, capturing it if we match.
743inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; }
744inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) {
745  return V;
746}
747 
748/// Match an arbitrary immediate Constant and ignore it.
749inline match_combine_and<class_match<Constant>,
750                         match_unless<constantexpr_match>>
751m_ImmConstant() {
752  return m_CombineAnd(m_Constant(), m_Unless(m_ConstantExpr()));
753}
754 
755/// Match an immediate Constant, capturing the value if we match.
756inline match_combine_and<bind_ty<Constant>,
757                         match_unless<constantexpr_match>>
758m_ImmConstant(Constant *&C) {
759  return m_CombineAnd(m_Constant(C), m_Unless(m_ConstantExpr()));
760}
761 
762/// Match a specified Value*.
763struct specificval_ty {
764  const Value *Val;
765 
766  specificval_ty(const Value *V) : Val(V) {}
767 
768  template <typename ITy> bool match(ITy *V) { return V == Val; }
769};
770 
771/// Match if we have a specific specified value.
772inline specificval_ty m_Specific(const Value *V) { return V; }
773 
774/// Stores a reference to the Value *, not the Value * itself,
775/// thus can be used in commutative matchers.
776template <typename Class> struct deferredval_ty {
777  Class *const &Val;
778 
779  deferredval_ty(Class *const &V) : Val(V) {}
780 
781  template <typename ITy> bool match(ITy *const V) { return V == Val; }
782};
783 
784/// Like m_Specific(), but works if the specific value to match is determined
785/// as part of the same match() expression. For example:
786/// m_Add(m_Value(X), m_Specific(X)) is incorrect, because m_Specific() will
787/// bind X before the pattern match starts.
788/// m_Add(m_Value(X), m_Deferred(X)) is correct, and will check against
789/// whichever value m_Value(X) populated.
790inline deferredval_ty<Value> m_Deferred(Value *const &V) { return V; }
791inline deferredval_ty<const Value> m_Deferred(const Value *const &V) {
792  return V;
793}
794 
795/// Match a specified floating point value or vector of all elements of
796/// that value.
797struct specific_fpval {
798  double Val;
799 
800  specific_fpval(double V) : Val(V) {}
801 
802  template <typename ITy> bool match(ITy *V) {
803    if (const auto *CFP = dyn_cast<ConstantFP>(V))
804      return CFP->isExactlyValue(Val);
805    if (V->getType()->isVectorTy())
806      if (const auto *C = dyn_cast<Constant>(V))
807        if (auto *CFP = dyn_cast_or_null<ConstantFP>(C->getSplatValue()))
808          return CFP->isExactlyValue(Val);
809    return false;
810  }
811};
812 
813/// Match a specific floating point value or vector with all elements
814/// equal to the value.
815inline specific_fpval m_SpecificFP(double V) { return specific_fpval(V); }
816 
817/// Match a float 1.0 or vector with all elements equal to 1.0.
818inline specific_fpval m_FPOne() { return m_SpecificFP(1.0); }
819 
820struct bind_const_intval_ty {
821  uint64_t &VR;
822 
823  bind_const_intval_ty(uint64_t &V) : VR(V) {}
824 
825  template <typename ITy> bool match(ITy *V) {
826    if (const auto *CV = dyn_cast<ConstantInt>(V))
827      if (CV->getValue().ule(UINT64_MAX(18446744073709551615UL))) {
828        VR = CV->getZExtValue();
829        return true;
830      }
831    return false;
832  }
833};
834 
835/// Match a specified integer value or vector of all elements of that
836/// value.
837template <bool AllowUndefs> struct specific_intval {
838  APInt Val;
839 
840  specific_intval(APInt V) : Val(std::move(V)) {}
841 
842  template <typename ITy> bool match(ITy *V) {
843    const auto *CI = dyn_cast<ConstantInt>(V);
844    if (!CI && V->getType()->isVectorTy())
845      if (const auto *C = dyn_cast<Constant>(V))
846        CI = dyn_cast_or_null<ConstantInt>(C->getSplatValue(AllowUndefs));
847 
848    return CI && APInt::isSameValue(CI->getValue(), Val);
849  }
850};
851 
852/// Match a specific integer value or vector with all elements equal to
853/// the value.
854inline specific_intval<false> m_SpecificInt(APInt V) {
855  return specific_intval<false>(std::move(V));
856}
857 
858inline specific_intval<false> m_SpecificInt(uint64_t V) {
859  return m_SpecificInt(APInt(64, V));
860}
861 
862inline specific_intval<true> m_SpecificIntAllowUndef(APInt V) {
863  return specific_intval<true>(std::move(V));
864}
865 
866inline specific_intval<true> m_SpecificIntAllowUndef(uint64_t V) {
867  return m_SpecificIntAllowUndef(APInt(64, V));
868}
869 
870/// Match a ConstantInt and bind to its value.  This does not match
871/// ConstantInts wider than 64-bits.
872inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; }
873 
874/// Match a specified basic block value.
875struct specific_bbval {
876  BasicBlock *Val;
877 
878  specific_bbval(BasicBlock *Val) : Val(Val) {}
879 
880  template <typename ITy> bool match(ITy *V) {
881    const auto *BB = dyn_cast<BasicBlock>(V);
882    return BB && BB == Val;
883  }
884};
885 
886/// Match a specific basic block value.
887inline specific_bbval m_SpecificBB(BasicBlock *BB) {
888  return specific_bbval(BB);
889}
890 
891/// A commutative-friendly version of m_Specific().
892inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) {
893  return BB;
894}
895inline deferredval_ty<const BasicBlock>
896m_Deferred(const BasicBlock *const &BB) {
897  return BB;
898}
899 
900//===----------------------------------------------------------------------===//
901// Matcher for any binary operator.
902//
903template <typename LHS_t, typename RHS_t, bool Commutable = false>
904struct AnyBinaryOp_match {
905  LHS_t L;
906  RHS_t R;
907 
908  // The evaluation order is always stable, regardless of Commutability.
909  // The LHS is always matched first.
910  AnyBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
911 
912  template <typename OpTy> bool match(OpTy *V) {
913    if (auto *I = dyn_cast<BinaryOperator>(V))
914      return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
915             (Commutable && L.match(I->getOperand(1)) &&
916              R.match(I->getOperand(0)));
917    return false;
918  }
919};
920 
921template <typename LHS, typename RHS>
922inline AnyBinaryOp_match<LHS, RHS> m_BinOp(const LHS &L, const RHS &R) {
923  return AnyBinaryOp_match<LHS, RHS>(L, R);
924}
925 
926//===----------------------------------------------------------------------===//
927// Matcher for any unary operator.
928// TODO fuse unary, binary matcher into n-ary matcher
929//
930template <typename OP_t> struct AnyUnaryOp_match {
931  OP_t X;
932 
933  AnyUnaryOp_match(const OP_t &X) : X(X) {}
934 
935  template <typename OpTy> bool match(OpTy *V) {
936    if (auto *I = dyn_cast<UnaryOperator>(V))
937      return X.match(I->getOperand(0));
938    return false;
939  }
940};
941 
942template <typename OP_t> inline AnyUnaryOp_match<OP_t> m_UnOp(const OP_t &X) {
943  return AnyUnaryOp_match<OP_t>(X);
944}
945 
946//===----------------------------------------------------------------------===//
947// Matchers for specific binary operators.
948//
949 
950template <typename LHS_t, typename RHS_t, unsigned Opcode,
951          bool Commutable = false>
952struct BinaryOp_match {
953  LHS_t L;
954  RHS_t R;
955 
956  // The evaluation order is always stable, regardless of Commutability.
957  // The LHS is always matched first.
958  BinaryOp_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
959 
960  template <typename OpTy> inline bool match(unsigned Opc, OpTy *V) {
961    if (V->getValueID() == Value::InstructionVal + Opc) {
77
←
Called C++ object pointer is null
962      auto *I = cast<BinaryOperator>(V);
963      return (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) ||
964             (Commutable && L.match(I->getOperand(1)) &&
965              R.match(I->getOperand(0)));
966    }
967    if (auto *CE = dyn_cast<ConstantExpr>(V))
968      return CE->getOpcode() == Opc &&
969             ((L.match(CE->getOperand(0)) && R.match(CE->getOperand(1))) ||
970              (Commutable && L.match(CE->getOperand(1)) &&
971               R.match(CE->getOperand(0))));
972    return false;
973  }
974 
975  template <typename OpTy> bool match(OpTy *V) { return match(Opcode, V); }
75
←
Passing null pointer value via 2nd parameter 'V'→
76
←
Calling 'BinaryOp_match::match'→
976};
977 
978template <typename LHS, typename RHS>
979inline BinaryOp_match<LHS, RHS, Instruction::Add> m_Add(const LHS &L,
980                                                        const RHS &R) {
981  return BinaryOp_match<LHS, RHS, Instruction::Add>(L, R);
982}
983 
984template <typename LHS, typename RHS>
985inline BinaryOp_match<LHS, RHS, Instruction::FAdd> m_FAdd(const LHS &L,
986                                                          const RHS &R) {
987  return BinaryOp_match<LHS, RHS, Instruction::FAdd>(L, R);
988}
989 
990template <typename LHS, typename RHS>
991inline BinaryOp_match<LHS, RHS, Instruction::Sub> m_Sub(const LHS &L,
992                                                        const RHS &R) {
993  return BinaryOp_match<LHS, RHS, Instruction::Sub>(L, R);
994}
995 
996template <typename LHS, typename RHS>
997inline BinaryOp_match<LHS, RHS, Instruction::FSub> m_FSub(const LHS &L,
998                                                          const RHS &R) {
999  return BinaryOp_match<LHS, RHS, Instruction::FSub>(L, R);
1000}
1001 
1002template <typename Op_t> struct FNeg_match {
1003  Op_t X;
1004 
1005  FNeg_match(const Op_t &Op) : X(Op) {}
1006  template <typename OpTy> bool match(OpTy *V) {
1007    auto *FPMO = dyn_cast<FPMathOperator>(V);
1008    if (!FPMO)
1009      return false;
1010 
1011    if (FPMO->getOpcode() == Instruction::FNeg)
1012      return X.match(FPMO->getOperand(0));
1013 
1014    if (FPMO->getOpcode() == Instruction::FSub) {
1015      if (FPMO->hasNoSignedZeros()) {
1016        // With 'nsz', any zero goes.
1017        if (!cstfp_pred_ty<is_any_zero_fp>().match(FPMO->getOperand(0)))
1018          return false;
1019      } else {
1020        // Without 'nsz', we need fsub -0.0, X exactly.
1021        if (!cstfp_pred_ty<is_neg_zero_fp>().match(FPMO->getOperand(0)))
1022          return false;
1023      }
1024 
1025      return X.match(FPMO->getOperand(1));
1026    }
1027 
1028    return false;
1029  }
1030};
1031 
1032/// Match 'fneg X' as 'fsub -0.0, X'.
1033template <typename OpTy> inline FNeg_match<OpTy> m_FNeg(const OpTy &X) {
1034  return FNeg_match<OpTy>(X);
1035}
1036 
1037/// Match 'fneg X' as 'fsub +-0.0, X'.
1038template <typename RHS>
1039inline BinaryOp_match<cstfp_pred_ty<is_any_zero_fp>, RHS, Instruction::FSub>
1040m_FNegNSZ(const RHS &X) {
1041  return m_FSub(m_AnyZeroFP(), X);
1042}
1043 
1044template <typename LHS, typename RHS>
1045inline BinaryOp_match<LHS, RHS, Instruction::Mul> m_Mul(const LHS &L,
1046                                                        const RHS &R) {
1047  return BinaryOp_match<LHS, RHS, Instruction::Mul>(L, R);
1048}
1049 
1050template <typename LHS, typename RHS>
1051inline BinaryOp_match<LHS, RHS, Instruction::FMul> m_FMul(const LHS &L,
1052                                                          const RHS &R) {
1053  return BinaryOp_match<LHS, RHS, Instruction::FMul>(L, R);
1054}
1055 
1056template <typename LHS, typename RHS>
1057inline BinaryOp_match<LHS, RHS, Instruction::UDiv> m_UDiv(const LHS &L,
1058                                                          const RHS &R) {
1059  return BinaryOp_match<LHS, RHS, Instruction::UDiv>(L, R);
1060}
1061 
1062template <typename LHS, typename RHS>
1063inline BinaryOp_match<LHS, RHS, Instruction::SDiv> m_SDiv(const LHS &L,
1064                                                          const RHS &R) {
1065  return BinaryOp_match<LHS, RHS, Instruction::SDiv>(L, R);
1066}
1067 
1068template <typename LHS, typename RHS>
1069inline BinaryOp_match<LHS, RHS, Instruction::FDiv> m_FDiv(const LHS &L,
1070                                                          const RHS &R) {
1071  return BinaryOp_match<LHS, RHS, Instruction::FDiv>(L, R);
1072}
1073 
1074template <typename LHS, typename RHS>
1075inline BinaryOp_match<LHS, RHS, Instruction::URem> m_URem(const LHS &L,
1076                                                          const RHS &R) {
1077  return BinaryOp_match<LHS, RHS, Instruction::URem>(L, R);
1078}
1079 
1080template <typename LHS, typename RHS>
1081inline BinaryOp_match<LHS, RHS, Instruction::SRem> m_SRem(const LHS &L,
1082                                                          const RHS &R) {
1083  return BinaryOp_match<LHS, RHS, Instruction::SRem>(L, R);
1084}
1085 
1086template <typename LHS, typename RHS>
1087inline BinaryOp_match<LHS, RHS, Instruction::FRem> m_FRem(const LHS &L,
1088                                                          const RHS &R) {
1089  return BinaryOp_match<LHS, RHS, Instruction::FRem>(L, R);
1090}
1091 
1092template <typename LHS, typename RHS>
1093inline BinaryOp_match<LHS, RHS, Instruction::And> m_And(const LHS &L,
1094                                                        const RHS &R) {
1095  return BinaryOp_match<LHS, RHS, Instruction::And>(L, R);
1096}
1097 
1098template <typename LHS, typename RHS>
1099inline BinaryOp_match<LHS, RHS, Instruction::Or> m_Or(const LHS &L,
1100                                                      const RHS &R) {
1101  return BinaryOp_match<LHS, RHS, Instruction::Or>(L, R);
1102}
1103 
1104template <typename LHS, typename RHS>
1105inline BinaryOp_match<LHS, RHS, Instruction::Xor> m_Xor(const LHS &L,
1106                                                        const RHS &R) {
1107  return BinaryOp_match<LHS, RHS, Instruction::Xor>(L, R);
1108}
1109 
1110template <typename LHS, typename RHS>
1111inline BinaryOp_match<LHS, RHS, Instruction::Shl> m_Shl(const LHS &L,
1112                                                        const RHS &R) {
1113  return BinaryOp_match<LHS, RHS, Instruction::Shl>(L, R);
1114}
1115 
1116template <typename LHS, typename RHS>
1117inline BinaryOp_match<LHS, RHS, Instruction::LShr> m_LShr(const LHS &L,
1118                                                          const RHS &R) {
1119  return BinaryOp_match<LHS, RHS, Instruction::LShr>(L, R);
1120}
1121 
1122template <typename LHS, typename RHS>
1123inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L,
1124                                                          const RHS &R) {
1125  return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R);
1126}
1127 
1128template <typename LHS_t, typename RHS_t, unsigned Opcode,
1129          unsigned WrapFlags = 0>
1130struct OverflowingBinaryOp_match {
1131  LHS_t L;
1132  RHS_t R;
1133 
1134  OverflowingBinaryOp_match(const LHS_t &LHS, const RHS_t &RHS)
1135      : L(LHS), R(RHS) {}
1136 
1137  template <typename OpTy> bool match(OpTy *V) {
1138    if (auto *Op = dyn_cast<OverflowingBinaryOperator>(V)) {
1139      if (Op->getOpcode() != Opcode)
1140        return false;
1141      if ((WrapFlags & OverflowingBinaryOperator::NoUnsignedWrap) &&
1142          !Op->hasNoUnsignedWrap())
1143        return false;
1144      if ((WrapFlags & OverflowingBinaryOperator::NoSignedWrap) &&
1145          !Op->hasNoSignedWrap())
1146        return false;
1147      return L.match(Op->getOperand(0)) && R.match(Op->getOperand(1));
1148    }
1149    return false;
1150  }
1151};
1152 
1153template <typename LHS, typename RHS>
1154inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1155                                 OverflowingBinaryOperator::NoSignedWrap>
1156m_NSWAdd(const LHS &L, const RHS &R) {
1157  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1158                                   OverflowingBinaryOperator::NoSignedWrap>(L,
1159                                                                            R);
1160}
1161template <typename LHS, typename RHS>
1162inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1163                                 OverflowingBinaryOperator::NoSignedWrap>
1164m_NSWSub(const LHS &L, const RHS &R) {
1165  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1166                                   OverflowingBinaryOperator::NoSignedWrap>(L,
1167                                                                            R);
1168}
1169template <typename LHS, typename RHS>
1170inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1171                                 OverflowingBinaryOperator::NoSignedWrap>
1172m_NSWMul(const LHS &L, const RHS &R) {
1173  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1174                                   OverflowingBinaryOperator::NoSignedWrap>(L,
1175                                                                            R);
1176}
1177template <typename LHS, typename RHS>
1178inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1179                                 OverflowingBinaryOperator::NoSignedWrap>
1180m_NSWShl(const LHS &L, const RHS &R) {
1181  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1182                                   OverflowingBinaryOperator::NoSignedWrap>(L,
1183                                                                            R);
1184}
1185 
1186template <typename LHS, typename RHS>
1187inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1188                                 OverflowingBinaryOperator::NoUnsignedWrap>
1189m_NUWAdd(const LHS &L, const RHS &R) {
1190  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
1191                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1192      L, R);
1193}
1194template <typename LHS, typename RHS>
1195inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1196                                 OverflowingBinaryOperator::NoUnsignedWrap>
1197m_NUWSub(const LHS &L, const RHS &R) {
1198  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
1199                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1200      L, R);
1201}
1202template <typename LHS, typename RHS>
1203inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1204                                 OverflowingBinaryOperator::NoUnsignedWrap>
1205m_NUWMul(const LHS &L, const RHS &R) {
1206  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Mul,
1207                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1208      L, R);
1209}
1210template <typename LHS, typename RHS>
1211inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1212                                 OverflowingBinaryOperator::NoUnsignedWrap>
1213m_NUWShl(const LHS &L, const RHS &R) {
1214  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Shl,
1215                                   OverflowingBinaryOperator::NoUnsignedWrap>(
1216      L, R);
1217}
1218 
1219template <typename LHS_t, typename RHS_t, bool Commutable = false>
1220struct SpecificBinaryOp_match
1221    : public BinaryOp_match<LHS_t, RHS_t, 0, Commutable> {
1222  unsigned Opcode;
1223 
1224  SpecificBinaryOp_match(unsigned Opcode, const LHS_t &LHS, const RHS_t &RHS)
1225      : BinaryOp_match<LHS_t, RHS_t, 0, Commutable>(LHS, RHS), Opcode(Opcode) {}
1226 
1227  template <typename OpTy> bool match(OpTy *V) {
1228    return BinaryOp_match<LHS_t, RHS_t, 0, Commutable>::match(Opcode, V);
1229  }
1230};
1231 
1232/// Matches a specific opcode.
1233template <typename LHS, typename RHS>
1234inline SpecificBinaryOp_match<LHS, RHS> m_BinOp(unsigned Opcode, const LHS &L,
1235                                                const RHS &R) {
1236  return SpecificBinaryOp_match<LHS, RHS>(Opcode, L, R);
1237}
1238 
1239//===----------------------------------------------------------------------===//
1240// Class that matches a group of binary opcodes.
1241//
1242template <typename LHS_t, typename RHS_t, typename Predicate>
1243struct BinOpPred_match : Predicate {
1244  LHS_t L;
1245  RHS_t R;
1246 
1247  BinOpPred_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
1248 
1249  template <typename OpTy> bool match(OpTy *V) {
1250    if (auto *I = dyn_cast<Instruction>(V))
1251      return this->isOpType(I->getOpcode()) && L.match(I->getOperand(0)) &&
1252             R.match(I->getOperand(1));
1253    if (auto *CE = dyn_cast<ConstantExpr>(V))
1254      return this->isOpType(CE->getOpcode()) && L.match(CE->getOperand(0)) &&
1255             R.match(CE->getOperand(1));
1256    return false;
1257  }
1258};
1259 
1260struct is_shift_op {
1261  bool isOpType(unsigned Opcode) { return Instruction::isShift(Opcode); }
1262};
1263 
1264struct is_right_shift_op {
1265  bool isOpType(unsigned Opcode) {
1266    return Opcode == Instruction::LShr || Opcode == Instruction::AShr;
1267  }
1268};
1269 
1270struct is_logical_shift_op {
1271  bool isOpType(unsigned Opcode) {
1272    return Opcode == Instruction::LShr || Opcode == Instruction::Shl;
1273  }
1274};
1275 
1276struct is_bitwiselogic_op {
1277  bool isOpType(unsigned Opcode) {
1278    return Instruction::isBitwiseLogicOp(Opcode);
1279  }
1280};
1281 
1282struct is_idiv_op {
1283  bool isOpType(unsigned Opcode) {
1284    return Opcode == Instruction::SDiv || Opcode == Instruction::UDiv;
1285  }
1286};
1287 
1288struct is_irem_op {
1289  bool isOpType(unsigned Opcode) {
1290    return Opcode == Instruction::SRem || Opcode == Instruction::URem;
1291  }
1292};
1293 
1294/// Matches shift operations.
1295template <typename LHS, typename RHS>
1296inline BinOpPred_match<LHS, RHS, is_shift_op> m_Shift(const LHS &L,
1297                                                      const RHS &R) {
1298  return BinOpPred_match<LHS, RHS, is_shift_op>(L, R);
1299}
1300 
1301/// Matches logical shift operations.
1302template <typename LHS, typename RHS>
1303inline BinOpPred_match<LHS, RHS, is_right_shift_op> m_Shr(const LHS &L,
1304                                                          const RHS &R) {
1305  return BinOpPred_match<LHS, RHS, is_right_shift_op>(L, R);
1306}
1307 
1308/// Matches logical shift operations.
1309template <typename LHS, typename RHS>
1310inline BinOpPred_match<LHS, RHS, is_logical_shift_op>
1311m_LogicalShift(const LHS &L, const RHS &R) {
1312  return BinOpPred_match<LHS, RHS, is_logical_shift_op>(L, R);
1313}
1314 
1315/// Matches bitwise logic operations.
1316template <typename LHS, typename RHS>
1317inline BinOpPred_match<LHS, RHS, is_bitwiselogic_op>
1318m_BitwiseLogic(const LHS &L, const RHS &R) {
1319  return BinOpPred_match<LHS, RHS, is_bitwiselogic_op>(L, R);
1320}
1321 
1322/// Matches integer division operations.
1323template <typename LHS, typename RHS>
1324inline BinOpPred_match<LHS, RHS, is_idiv_op> m_IDiv(const LHS &L,
1325                                                    const RHS &R) {
1326  return BinOpPred_match<LHS, RHS, is_idiv_op>(L, R);
1327}
1328 
1329/// Matches integer remainder operations.
1330template <typename LHS, typename RHS>
1331inline BinOpPred_match<LHS, RHS, is_irem_op> m_IRem(const LHS &L,
1332                                                    const RHS &R) {
1333  return BinOpPred_match<LHS, RHS, is_irem_op>(L, R);
1334}
1335 
1336//===----------------------------------------------------------------------===//
1337// Class that matches exact binary ops.
1338//
1339template <typename SubPattern_t> struct Exact_match {
1340  SubPattern_t SubPattern;
1341 
1342  Exact_match(const SubPattern_t &SP) : SubPattern(SP) {}
1343 
1344  template <typename OpTy> bool match(OpTy *V) {
1345    if (auto *PEO = dyn_cast<PossiblyExactOperator>(V))
1346      return PEO->isExact() && SubPattern.match(V);
1347    return false;
1348  }
1349};
1350 
1351template <typename T> inline Exact_match<T> m_Exact(const T &SubPattern) {
1352  return SubPattern;
1353}
1354 
1355//===----------------------------------------------------------------------===//
1356// Matchers for CmpInst classes
1357//
1358 
1359template <typename LHS_t, typename RHS_t, typename Class, typename PredicateTy,
1360          bool Commutable = false>
1361struct CmpClass_match {
1362  PredicateTy &Predicate;
1363  LHS_t L;
1364  RHS_t R;
1365 
1366  // The evaluation order is always stable, regardless of Commutability.
1367  // The LHS is always matched first.
1368  CmpClass_match(PredicateTy &Pred, const LHS_t &LHS, const RHS_t &RHS)
1369      : Predicate(Pred), L(LHS), R(RHS) {}
1370 
1371  template <typename OpTy> bool match(OpTy *V) {
1372    if (auto *I = dyn_cast<Class>(V)) {
1373      if (L.match(I->getOperand(0)) && R.match(I->getOperand(1))) {
1374        Predicate = I->getPredicate();
1375        return true;
1376      } else if (Commutable && L.match(I->getOperand(1)) &&
1377                 R.match(I->getOperand(0))) {
1378        Predicate = I->getSwappedPredicate();
1379        return true;
1380      }
1381    }
1382    return false;
1383  }
1384};
1385 
1386template <typename LHS, typename RHS>
1387inline CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>
1388m_Cmp(CmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1389  return CmpClass_match<LHS, RHS, CmpInst, CmpInst::Predicate>(Pred, L, R);
1390}
1391 
1392template <typename LHS, typename RHS>
1393inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>
1394m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1395  return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate>(Pred, L, R);
1396}
1397 
1398template <typename LHS, typename RHS>
1399inline CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>
1400m_FCmp(FCmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
1401  return CmpClass_match<LHS, RHS, FCmpInst, FCmpInst::Predicate>(Pred, L, R);
1402}
1403 
1404//===----------------------------------------------------------------------===//
1405// Matchers for instructions with a given opcode and number of operands.
1406//
1407 
1408/// Matches instructions with Opcode and three operands.
1409template <typename T0, unsigned Opcode> struct OneOps_match {
1410  T0 Op1;
1411 
1412  OneOps_match(const T0 &Op1) : Op1(Op1) {}
1413 
1414  template <typename OpTy> bool match(OpTy *V) {
1415    if (V->getValueID() == Value::InstructionVal + Opcode) {
1416      auto *I = cast<Instruction>(V);
1417      return Op1.match(I->getOperand(0));
1418    }
1419    return false;
1420  }
1421};
1422 
1423/// Matches instructions with Opcode and three operands.
1424template <typename T0, typename T1, unsigned Opcode> struct TwoOps_match {
1425  T0 Op1;
1426  T1 Op2;
1427 
1428  TwoOps_match(const T0 &Op1, const T1 &Op2) : Op1(Op1), Op2(Op2) {}
1429 
1430  template <typename OpTy> bool match(OpTy *V) {
1431    if (V->getValueID() == Value::InstructionVal + Opcode) {
1432      auto *I = cast<Instruction>(V);
1433      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1));
1434    }
1435    return false;
1436  }
1437};
1438 
1439/// Matches instructions with Opcode and three operands.
1440template <typename T0, typename T1, typename T2, unsigned Opcode>
1441struct ThreeOps_match {
1442  T0 Op1;
1443  T1 Op2;
1444  T2 Op3;
1445 
1446  ThreeOps_match(const T0 &Op1, const T1 &Op2, const T2 &Op3)
1447      : Op1(Op1), Op2(Op2), Op3(Op3) {}
1448 
1449  template <typename OpTy> bool match(OpTy *V) {
1450    if (V->getValueID() == Value::InstructionVal + Opcode) {
1451      auto *I = cast<Instruction>(V);
1452      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
1453             Op3.match(I->getOperand(2));
1454    }
1455    return false;
1456  }
1457};
1458 
1459/// Matches SelectInst.
1460template <typename Cond, typename LHS, typename RHS>
1461inline ThreeOps_match<Cond, LHS, RHS, Instruction::Select>
1462m_Select(const Cond &C, const LHS &L, const RHS &R) {
1463  return ThreeOps_match<Cond, LHS, RHS, Instruction::Select>(C, L, R);
55
←
Returning without writing to 'C.VR'→
1464}
1465 
1466/// This matches a select of two constants, e.g.:
1467/// m_SelectCst<-1, 0>(m_Value(V))
1468template <int64_t L, int64_t R, typename Cond>
1469inline ThreeOps_match<Cond, constantint_match<L>, constantint_match<R>,
1470                      Instruction::Select>
1471m_SelectCst(const Cond &C) {
1472  return m_Select(C, m_ConstantInt<L>(), m_ConstantInt<R>());
1473}
1474 
1475/// Matches FreezeInst.
1476template <typename OpTy>
1477inline OneOps_match<OpTy, Instruction::Freeze> m_Freeze(const OpTy &Op) {
1478  return OneOps_match<OpTy, Instruction::Freeze>(Op);
1479}
1480 
1481/// Matches InsertElementInst.
1482template <typename Val_t, typename Elt_t, typename Idx_t>
1483inline ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>
1484m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx) {
1485  return ThreeOps_match<Val_t, Elt_t, Idx_t, Instruction::InsertElement>(
1486      Val, Elt, Idx);
1487}
1488 
1489/// Matches ExtractElementInst.
1490template <typename Val_t, typename Idx_t>
1491inline TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>
1492m_ExtractElt(const Val_t &Val, const Idx_t &Idx) {
1493  return TwoOps_match<Val_t, Idx_t, Instruction::ExtractElement>(Val, Idx);
1494}
1495 
1496/// Matches shuffle.
1497template <typename T0, typename T1, typename T2> struct Shuffle_match {
1498  T0 Op1;
1499  T1 Op2;
1500  T2 Mask;
1501 
1502  Shuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
1503      : Op1(Op1), Op2(Op2), Mask(Mask) {}
1504 
1505  template <typename OpTy> bool match(OpTy *V) {
1506    if (auto *I = dyn_cast<ShuffleVectorInst>(V)) {
1507      return Op1.match(I->getOperand(0)) && Op2.match(I->getOperand(1)) &&
1508             Mask.match(I->getShuffleMask());
1509    }
1510    return false;
1511  }
1512};
1513 
1514struct m_Mask {
1515  ArrayRef<int> &MaskRef;
1516  m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
1517  bool match(ArrayRef<int> Mask) {
1518    MaskRef = Mask;
1519    return true;
1520  }
1521};
1522 
1523struct m_ZeroMask {
1524  bool match(ArrayRef<int> Mask) {
1525    return all_of(Mask, [](int Elem) { return Elem == 0 || Elem == -1; });
1526  }
1527};
1528 
1529struct m_SpecificMask {
1530  ArrayRef<int> &MaskRef;
1531  m_SpecificMask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
1532  bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
1533};
1534 
1535struct m_SplatOrUndefMask {
1536  int &SplatIndex;
1537  m_SplatOrUndefMask(int &SplatIndex) : SplatIndex(SplatIndex) {}
1538  bool match(ArrayRef<int> Mask) {
1539    const auto *First = find_if(Mask, [](int Elem) { return Elem != -1; });
1540    if (First == Mask.end())
1541      return false;
1542    SplatIndex = *First;
1543    return all_of(Mask,
1544                  [First](int Elem) { return Elem == *First || Elem == -1; });
1545  }
1546};
1547 
1548/// Matches ShuffleVectorInst independently of mask value.
1549template <typename V1_t, typename V2_t>
1550inline TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>
1551m_Shuffle(const V1_t &v1, const V2_t &v2) {
1552  return TwoOps_match<V1_t, V2_t, Instruction::ShuffleVector>(v1, v2);
1553}
1554 
1555template <typename V1_t, typename V2_t, typename Mask_t>
1556inline Shuffle_match<V1_t, V2_t, Mask_t>
1557m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
1558  return Shuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
1559}
1560 
1561/// Matches LoadInst.
1562template <typename OpTy>
1563inline OneOps_match<OpTy, Instruction::Load> m_Load(const OpTy &Op) {
1564  return OneOps_match<OpTy, Instruction::Load>(Op);
1565}
1566 
1567/// Matches StoreInst.
1568template <typename ValueOpTy, typename PointerOpTy>
1569inline TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>
1570m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp) {
1571  return TwoOps_match<ValueOpTy, PointerOpTy, Instruction::Store>(ValueOp,
1572                                                                  PointerOp);
1573}
1574 
1575//===----------------------------------------------------------------------===//
1576// Matchers for CastInst classes
1577//
1578 
1579template <typename Op_t, unsigned Opcode> struct CastClass_match {
1580  Op_t Op;
1581 
1582  CastClass_match(const Op_t &OpMatch) : Op(OpMatch) {}
1583 
1584  template <typename OpTy> bool match(OpTy *V) {
1585    if (auto *O = dyn_cast<Operator>(V))
1586      return O->getOpcode() == Opcode && Op.match(O->getOperand(0));
1587    return false;
1588  }
1589};
1590 
1591/// Matches BitCast.
1592template <typename OpTy>
1593inline CastClass_match<OpTy, Instruction::BitCast> m_BitCast(const OpTy &Op) {
1594  return CastClass_match<OpTy, Instruction::BitCast>(Op);
1595}
1596 
1597/// Matches PtrToInt.
1598template <typename OpTy>
1599inline CastClass_match<OpTy, Instruction::PtrToInt> m_PtrToInt(const OpTy &Op) {
1600  return CastClass_match<OpTy, Instruction::PtrToInt>(Op);
1601}
1602 
1603/// Matches IntToPtr.
1604template <typename OpTy>
1605inline CastClass_match<OpTy, Instruction::IntToPtr> m_IntToPtr(const OpTy &Op) {
1606  return CastClass_match<OpTy, Instruction::IntToPtr>(Op);
1607}
1608 
1609/// Matches Trunc.
1610template <typename OpTy>
1611inline CastClass_match<OpTy, Instruction::Trunc> m_Trunc(const OpTy &Op) {
1612  return CastClass_match<OpTy, Instruction::Trunc>(Op);
1613}
1614 
1615template <typename OpTy>
1616inline match_combine_or<CastClass_match<OpTy, Instruction::Trunc>, OpTy>
1617m_TruncOrSelf(const OpTy &Op) {
1618  return m_CombineOr(m_Trunc(Op), Op);
1619}
1620 
1621/// Matches SExt.
1622template <typename OpTy>
1623inline CastClass_match<OpTy, Instruction::SExt> m_SExt(const OpTy &Op) {
1624  return CastClass_match<OpTy, Instruction::SExt>(Op);
1625}
1626 
1627/// Matches ZExt.
1628template <typename OpTy>
1629inline CastClass_match<OpTy, Instruction::ZExt> m_ZExt(const OpTy &Op) {
1630  return CastClass_match<OpTy, Instruction::ZExt>(Op);
1631}
1632 
1633template <typename OpTy>
1634inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>, OpTy>
1635m_ZExtOrSelf(const OpTy &Op) {
1636  return m_CombineOr(m_ZExt(Op), Op);
1637}
1638 
1639template <typename OpTy>
1640inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy>
1641m_SExtOrSelf(const OpTy &Op) {
1642  return m_CombineOr(m_SExt(Op), Op);
1643}
1644 
1645template <typename OpTy>
1646inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
1647                        CastClass_match<OpTy, Instruction::SExt>>
1648m_ZExtOrSExt(const OpTy &Op) {
1649  return m_CombineOr(m_ZExt(Op), m_SExt(Op));
1650}
1651 
1652template <typename OpTy>
1653inline match_combine_or<
1654    match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
1655                     CastClass_match<OpTy, Instruction::SExt>>,
1656    OpTy>
1657m_ZExtOrSExtOrSelf(const OpTy &Op) {
1658  return m_CombineOr(m_ZExtOrSExt(Op), Op);
1659}
1660 
1661template <typename OpTy>
1662inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) {
1663  return CastClass_match<OpTy, Instruction::UIToFP>(Op);
1664}
1665 
1666template <typename OpTy>
1667inline CastClass_match<OpTy, Instruction::SIToFP> m_SIToFP(const OpTy &Op) {
1668  return CastClass_match<OpTy, Instruction::SIToFP>(Op);
1669}
1670 
1671template <typename OpTy>
1672inline CastClass_match<OpTy, Instruction::FPToUI> m_FPToUI(const OpTy &Op) {
1673  return CastClass_match<OpTy, Instruction::FPToUI>(Op);
1674}
1675 
1676template <typename OpTy>
1677inline CastClass_match<OpTy, Instruction::FPToSI> m_FPToSI(const OpTy &Op) {
1678  return CastClass_match<OpTy, Instruction::FPToSI>(Op);
1679}
1680 
1681template <typename OpTy>
1682inline CastClass_match<OpTy, Instruction::FPTrunc> m_FPTrunc(const OpTy &Op) {
1683  return CastClass_match<OpTy, Instruction::FPTrunc>(Op);
1684}
1685 
1686template <typename OpTy>
1687inline CastClass_match<OpTy, Instruction::FPExt> m_FPExt(const OpTy &Op) {
1688  return CastClass_match<OpTy, Instruction::FPExt>(Op);
1689}
1690 
1691//===----------------------------------------------------------------------===//
1692// Matchers for control flow.
1693//
1694 
1695struct br_match {
1696  BasicBlock *&Succ;
1697 
1698  br_match(BasicBlock *&Succ) : Succ(Succ) {}
1699 
1700  template <typename OpTy> bool match(OpTy *V) {
1701    if (auto *BI = dyn_cast<BranchInst>(V))
1702      if (BI->isUnconditional()) {
1703        Succ = BI->getSuccessor(0);
1704        return true;
1705      }
1706    return false;
1707  }
1708};
1709 
1710inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); }
1711 
1712template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
1713struct brc_match {
1714  Cond_t Cond;
1715  TrueBlock_t T;
1716  FalseBlock_t F;
1717 
1718  brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f)
1719      : Cond(C), T(t), F(f) {}
1720 
1721  template <typename OpTy> bool match(OpTy *V) {
1722    if (auto *BI = dyn_cast<BranchInst>(V))
1723      if (BI->isConditional() && Cond.match(BI->getCondition()))
1724        return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1));
1725    return false;
1726  }
1727};
1728 
1729template <typename Cond_t>
1730inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>
1731m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) {
1732  return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>(
1733      C, m_BasicBlock(T), m_BasicBlock(F));
1734}
1735 
1736template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
1737inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t>
1738m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) {
1739  return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F);
1740}
1741 
1742//===----------------------------------------------------------------------===//
1743// Matchers for max/min idioms, eg: "select (sgt x, y), x, y" -> smax(x,y).
1744//
1745 
1746template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t,
1747          bool Commutable = false>
1748struct MaxMin_match {
1749  using PredType = Pred_t;
1750  LHS_t L;
1751  RHS_t R;
1752 
1753  // The evaluation order is always stable, regardless of Commutability.
1754  // The LHS is always matched first.
1755  MaxMin_match(const LHS_t &LHS, const RHS_t &RHS) : L(LHS), R(RHS) {}
1756 
1757  template <typename OpTy> bool match(OpTy *V) {
1758    if (auto *II = dyn_cast<IntrinsicInst>(V)) {
1759      Intrinsic::ID IID = II->getIntrinsicID();
1760      if ((IID == Intrinsic::smax && Pred_t::match(ICmpInst::ICMP_SGT)) ||
1761          (IID == Intrinsic::smin && Pred_t::match(ICmpInst::ICMP_SLT)) ||
1762          (IID == Intrinsic::umax && Pred_t::match(ICmpInst::ICMP_UGT)) ||
1763          (IID == Intrinsic::umin && Pred_t::match(ICmpInst::ICMP_ULT))) {
1764        Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);
1765        return (L.match(LHS) && R.match(RHS)) ||
1766               (Commutable && L.match(RHS) && R.match(LHS));
1767      }
1768    }
1769    // Look for "(x pred y) ? x : y" or "(x pred y) ? y : x".
1770    auto *SI = dyn_cast<SelectInst>(V);
1771    if (!SI)
1772      return false;
1773    auto *Cmp = dyn_cast<CmpInst_t>(SI->getCondition());
1774    if (!Cmp)
1775      return false;
1776    // At this point we have a select conditioned on a comparison.  Check that
1777    // it is the values returned by the select that are being compared.
1778    auto *TrueVal = SI->getTrueValue();
1779    auto *FalseVal = SI->getFalseValue();
1780    auto *LHS = Cmp->getOperand(0);
1781    auto *RHS = Cmp->getOperand(1);
1782    if ((TrueVal != LHS || FalseVal != RHS) &&
1783        (TrueVal != RHS || FalseVal != LHS))
1784      return false;
1785    typename CmpInst_t::Predicate Pred =
1786        LHS == TrueVal ? Cmp->getPredicate() : Cmp->getInversePredicate();
1787    // Does "(x pred y) ? x : y" represent the desired max/min operation?
1788    if (!Pred_t::match(Pred))
1789      return false;
1790    // It does!  Bind the operands.
1791    return (L.match(LHS) && R.match(RHS)) ||
1792           (Commutable && L.match(RHS) && R.match(LHS));
1793  }
1794};
1795 
1796/// Helper class for identifying signed max predicates.
1797struct smax_pred_ty {
1798  static bool match(ICmpInst::Predicate Pred) {
1799    return Pred == CmpInst::ICMP_SGT || Pred == CmpInst::ICMP_SGE;
1800  }
1801};
1802 
1803/// Helper class for identifying signed min predicates.
1804struct smin_pred_ty {
1805  static bool match(ICmpInst::Predicate Pred) {
1806    return Pred == CmpInst::ICMP_SLT || Pred == CmpInst::ICMP_SLE;
1807  }
1808};
1809 
1810/// Helper class for identifying unsigned max predicates.
1811struct umax_pred_ty {
1812  static bool match(ICmpInst::Predicate Pred) {
1813    return Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE;
1814  }
1815};
1816 
1817/// Helper class for identifying unsigned min predicates.
1818struct umin_pred_ty {
1819  static bool match(ICmpInst::Predicate Pred) {
1820    return Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_ULE;
1821  }
1822};
1823 
1824/// Helper class for identifying ordered max predicates.
1825struct ofmax_pred_ty {
1826  static bool match(FCmpInst::Predicate Pred) {
1827    return Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_OGE;
1828  }
1829};
1830 
1831/// Helper class for identifying ordered min predicates.
1832struct ofmin_pred_ty {
1833  static bool match(FCmpInst::Predicate Pred) {
1834    return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE;
1835  }
1836};
1837 
1838/// Helper class for identifying unordered max predicates.
1839struct ufmax_pred_ty {
1840  static bool match(FCmpInst::Predicate Pred) {
1841    return Pred == CmpInst::FCMP_UGT || Pred == CmpInst::FCMP_UGE;
1842  }
1843};
1844 
1845/// Helper class for identifying unordered min predicates.
1846struct ufmin_pred_ty {
1847  static bool match(FCmpInst::Predicate Pred) {
1848    return Pred == CmpInst::FCMP_ULT || Pred == CmpInst::FCMP_ULE;
1849  }
1850};
1851 
1852template <typename LHS, typename RHS>
1853inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty> m_SMax(const LHS &L,
1854                                                             const RHS &R) {
1855  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>(L, R);
1856}
1857 
1858template <typename LHS, typename RHS>
1859inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty> m_SMin(const LHS &L,
1860                                                             const RHS &R) {
1861  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>(L, R);
1862}
1863 
1864template <typename LHS, typename RHS>
1865inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty> m_UMax(const LHS &L,
1866                                                             const RHS &R) {
1867  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>(L, R);
1868}
1869 
1870template <typename LHS, typename RHS>
1871inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty> m_UMin(const LHS &L,
1872                                                             const RHS &R) {
1873  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>(L, R);
1874}
1875 
1876template <typename LHS, typename RHS>
1877inline match_combine_or<
1878    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty>,
1879                     MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty>>,
1880    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty>,
1881                     MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty>>>
1882m_MaxOrMin(const LHS &L, const RHS &R) {
1883  return m_CombineOr(m_CombineOr(m_SMax(L, R), m_SMin(L, R)),
1884                     m_CombineOr(m_UMax(L, R), m_UMin(L, R)));
1885}
1886 
1887/// Match an 'ordered' floating point maximum function.
1888/// Floating point has one special value 'NaN'. Therefore, there is no total
1889/// order. However, if we can ignore the 'NaN' value (for example, because of a
1890/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
1891/// semantics. In the presence of 'NaN' we have to preserve the original
1892/// select(fcmp(ogt/ge, L, R), L, R) semantics matched by this predicate.
1893///
1894///                         max(L, R)  iff L and R are not NaN
1895///  m_OrdFMax(L, R) =      R          iff L or R are NaN
1896template <typename LHS, typename RHS>
1897inline MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty> m_OrdFMax(const LHS &L,
1898                                                                 const RHS &R) {
1899  return MaxMin_match<FCmpInst, LHS, RHS, ofmax_pred_ty>(L, R);
1900}
1901 
1902/// Match an 'ordered' floating point minimum function.
1903/// Floating point has one special value 'NaN'. Therefore, there is no total
1904/// order. However, if we can ignore the 'NaN' value (for example, because of a
1905/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
1906/// semantics. In the presence of 'NaN' we have to preserve the original
1907/// select(fcmp(olt/le, L, R), L, R) semantics matched by this predicate.
1908///
1909///                         min(L, R)  iff L and R are not NaN
1910///  m_OrdFMin(L, R) =      R          iff L or R are NaN
1911template <typename LHS, typename RHS>
1912inline MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty> m_OrdFMin(const LHS &L,
1913                                                                 const RHS &R) {
1914  return MaxMin_match<FCmpInst, LHS, RHS, ofmin_pred_ty>(L, R);
1915}
1916 
1917/// Match an 'unordered' floating point maximum function.
1918/// Floating point has one special value 'NaN'. Therefore, there is no total
1919/// order. However, if we can ignore the 'NaN' value (for example, because of a
1920/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'maximum'
1921/// semantics. In the presence of 'NaN' we have to preserve the original
1922/// select(fcmp(ugt/ge, L, R), L, R) semantics matched by this predicate.
1923///
1924///                         max(L, R)  iff L and R are not NaN
1925///  m_UnordFMax(L, R) =    L          iff L or R are NaN
1926template <typename LHS, typename RHS>
1927inline MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>
1928m_UnordFMax(const LHS &L, const RHS &R) {
1929  return MaxMin_match<FCmpInst, LHS, RHS, ufmax_pred_ty>(L, R);
1930}
1931 
1932/// Match an 'unordered' floating point minimum function.
1933/// Floating point has one special value 'NaN'. Therefore, there is no total
1934/// order. However, if we can ignore the 'NaN' value (for example, because of a
1935/// 'no-nans-float-math' flag) a combination of a fcmp and select has 'minimum'
1936/// semantics. In the presence of 'NaN' we have to preserve the original
1937/// select(fcmp(ult/le, L, R), L, R) semantics matched by this predicate.
1938///
1939///                          min(L, R)  iff L and R are not NaN
1940///  m_UnordFMin(L, R) =     L          iff L or R are NaN
1941template <typename LHS, typename RHS>
1942inline MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>
1943m_UnordFMin(const LHS &L, const RHS &R) {
1944  return MaxMin_match<FCmpInst, LHS, RHS, ufmin_pred_ty>(L, R);
1945}
1946 
1947//===----------------------------------------------------------------------===//
1948// Matchers for overflow check patterns: e.g. (a + b) u< a, (a ^ -1) <u b
1949// Note that S might be matched to other instructions than AddInst.
1950//
1951 
1952template <typename LHS_t, typename RHS_t, typename Sum_t>
1953struct UAddWithOverflow_match {
1954  LHS_t L;
1955  RHS_t R;
1956  Sum_t S;
1957 
1958  UAddWithOverflow_match(const LHS_t &L, const RHS_t &R, const Sum_t &S)
1959      : L(L), R(R), S(S) {}
1960 
1961  template <typename OpTy> bool match(OpTy *V) {
1962    Value *ICmpLHS, *ICmpRHS;
1963    ICmpInst::Predicate Pred;
1964    if (!m_ICmp(Pred, m_Value(ICmpLHS), m_Value(ICmpRHS)).match(V))
1965      return false;
1966 
1967    Value *AddLHS, *AddRHS;
1968    auto AddExpr = m_Add(m_Value(AddLHS), m_Value(AddRHS));
1969 
1970    // (a + b) u< a, (a + b) u< b
1971    if (Pred == ICmpInst::ICMP_ULT)
1972      if (AddExpr.match(ICmpLHS) && (ICmpRHS == AddLHS || ICmpRHS == AddRHS))
1973        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
1974 
1975    // a >u (a + b), b >u (a + b)
1976    if (Pred == ICmpInst::ICMP_UGT)
1977      if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS))
1978        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
1979 
1980    Value *Op1;
1981    auto XorExpr = m_OneUse(m_Xor(m_Value(Op1), m_AllOnes()));
1982    // (a ^ -1) <u b
1983    if (Pred == ICmpInst::ICMP_ULT) {
1984      if (XorExpr.match(ICmpLHS))
1985        return L.match(Op1) && R.match(ICmpRHS) && S.match(ICmpLHS);
1986    }
1987    //  b > u (a ^ -1)
1988    if (Pred == ICmpInst::ICMP_UGT) {
1989      if (XorExpr.match(ICmpRHS))
1990        return L.match(Op1) && R.match(ICmpLHS) && S.match(ICmpRHS);
1991    }
1992 
1993    // Match special-case for increment-by-1.
1994    if (Pred == ICmpInst::ICMP_EQ) {
1995      // (a + 1) == 0
1996      // (1 + a) == 0
1997      if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) &&
1998          (m_One().match(AddLHS) || m_One().match(AddRHS)))
1999        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
2000      // 0 == (a + 1)
2001      // 0 == (1 + a)
2002      if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) &&
2003          (m_One().match(AddLHS) || m_One().match(AddRHS)))
2004        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
2005    }
2006 
2007    return false;
2008  }
2009};
2010 
2011/// Match an icmp instruction checking for unsigned overflow on addition.
2012///
2013/// S is matched to the addition whose result is being checked for overflow, and
2014/// L and R are matched to the LHS and RHS of S.
2015template <typename LHS_t, typename RHS_t, typename Sum_t>
2016UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>
2017m_UAddWithOverflow(const LHS_t &L, const RHS_t &R, const Sum_t &S) {
2018  return UAddWithOverflow_match<LHS_t, RHS_t, Sum_t>(L, R, S);
2019}
2020 
2021template <typename Opnd_t> struct Argument_match {
2022  unsigned OpI;
2023  Opnd_t Val;
2024 
2025  Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
2026 
2027  template <typename OpTy> bool match(OpTy *V) {
2028    // FIXME: Should likely be switched to use `CallBase`.
2029    if (const auto *CI = dyn_cast<CallInst>(V))
2030      return Val.match(CI->getArgOperand(OpI));
2031    return false;
2032  }
2033};
2034 
2035/// Match an argument.
2036template <unsigned OpI, typename Opnd_t>
2037inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
2038  return Argument_match<Opnd_t>(OpI, Op);
2039}
2040 
2041/// Intrinsic matchers.
2042struct IntrinsicID_match {
2043  unsigned ID;
2044 
2045  IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
2046 
2047  template <typename OpTy> bool match(OpTy *V) {
2048    if (const auto *CI = dyn_cast<CallInst>(V))
2049      if (const auto *F = CI->getCalledFunction())
2050        return F->getIntrinsicID() == ID;
2051    return false;
2052  }
2053};
2054 
2055/// Intrinsic matches are combinations of ID matchers, and argument
2056/// matchers. Higher arity matcher are defined recursively in terms of and-ing
2057/// them with lower arity matchers. Here's some convenient typedefs for up to
2058/// several arguments, and more can be added as needed
2059template <typename T0 = void, typename T1 = void, typename T2 = void,
2060          typename T3 = void, typename T4 = void, typename T5 = void,
2061          typename T6 = void, typename T7 = void, typename T8 = void,
2062          typename T9 = void, typename T10 = void>
2063struct m_Intrinsic_Ty;
2064template <typename T0> struct m_Intrinsic_Ty<T0> {
2065  using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
2066};
2067template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
2068  using Ty =
2069      match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
2070};
2071template <typename T0, typename T1, typename T2>
2072struct m_Intrinsic_Ty<T0, T1, T2> {
2073  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
2074                               Argument_match<T2>>;
2075};
2076template <typename T0, typename T1, typename T2, typename T3>
2077struct m_Intrinsic_Ty<T0, T1, T2, T3> {
2078  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
2079                               Argument_match<T3>>;
2080};
2081 
2082template <typename T0, typename T1, typename T2, typename T3, typename T4>
2083struct m_Intrinsic_Ty<T0, T1, T2, T3, T4> {
2084  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty,
2085                               Argument_match<T4>>;
2086};
2087 
2088template <typename T0, typename T1, typename T2, typename T3, typename T4,
2089          typename T5>
2090struct m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5> {
2091  using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty,
2092                               Argument_match<T5>>;
2093};
2094 
2095/// Match intrinsic calls like this:
2096/// m_Intrinsic<Intrinsic::fabs>(m_Value(X))
2097template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
2098  return IntrinsicID_match(IntrID);
2099}
2100 
2101/// Matches MaskedLoad Intrinsic.
2102template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3>
2103inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty
2104m_MaskedLoad(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2,
2105             const Opnd3 &Op3) {
2106  return m_Intrinsic<Intrinsic::masked_load>(Op0, Op1, Op2, Op3);
2107}
2108 
2109/// Matches MaskedGather Intrinsic.
2110template <typename Opnd0, typename Opnd1, typename Opnd2, typename Opnd3>
2111inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2, Opnd3>::Ty
2112m_MaskedGather(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2,
2113               const Opnd3 &Op3) {
2114  return m_Intrinsic<Intrinsic::masked_gather>(Op0, Op1, Op2, Op3);
2115}
2116 
2117template <Intrinsic::ID IntrID, typename T0>
2118inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
2119  return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
2120}
2121 
2122template <Intrinsic::ID IntrID, typename T0, typename T1>
2123inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
2124                                                       const T1 &Op1) {
2125  return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
2126}
2127 
2128template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
2129inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
2130m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
2131  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
2132}
2133 
2134template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2135          typename T3>
2136inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
2137m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
2138  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
2139}
2140 
2141template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2142          typename T3, typename T4>
2143inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4>::Ty
2144m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
2145            const T4 &Op4) {
2146  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3),
2147                      m_Argument<4>(Op4));
2148}
2149 
2150template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
2151          typename T3, typename T4, typename T5>
2152inline typename m_Intrinsic_Ty<T0, T1, T2, T3, T4, T5>::Ty
2153m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3,
2154            const T4 &Op4, const T5 &Op5) {
2155  return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2, Op3, Op4),
2156                      m_Argument<5>(Op5));
2157}
2158 
2159// Helper intrinsic matching specializations.
2160template <typename Opnd0>
2161inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BitReverse(const Opnd0 &Op0) {
2162  return m_Intrinsic<Intrinsic::bitreverse>(Op0);
2163}
2164 
2165template <typename Opnd0>
2166inline typename m_Intrinsic_Ty<Opnd0>::Ty m_BSwap(const Opnd0 &Op0) {
2167  return m_Intrinsic<Intrinsic::bswap>(Op0);
2168}
2169 
2170template <typename Opnd0>
2171inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FAbs(const Opnd0 &Op0) {
2172  return m_Intrinsic<Intrinsic::fabs>(Op0);
2173}
2174 
2175template <typename Opnd0>
2176inline typename m_Intrinsic_Ty<Opnd0>::Ty m_FCanonicalize(const Opnd0 &Op0) {
2177  return m_Intrinsic<Intrinsic::canonicalize>(Op0);
2178}
2179 
2180template <typename Opnd0, typename Opnd1>
2181inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMin(const Opnd0 &Op0,
2182                                                        const Opnd1 &Op1) {
2183  return m_Intrinsic<Intrinsic::minnum>(Op0, Op1);
2184}
2185 
2186template <typename Opnd0, typename Opnd1>
2187inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_FMax(const Opnd0 &Op0,
2188                                                        const Opnd1 &Op1) {
2189  return m_Intrinsic<Intrinsic::maxnum>(Op0, Op1);
2190}
2191 
2192template <typename Opnd0, typename Opnd1, typename Opnd2>
2193inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
2194m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
2195  return m_Intrinsic<Intrinsic::fshl>(Op0, Op1, Op2);
2196}
2197 
2198template <typename Opnd0, typename Opnd1, typename Opnd2>
2199inline typename m_Intrinsic_Ty<Opnd0, Opnd1, Opnd2>::Ty
2200m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2) {
2201  return m_Intrinsic<Intrinsic::fshr>(Op0, Op1, Op2);
2202}
2203 
2204template <typename Opnd0>
2205inline typename m_Intrinsic_Ty<Opnd0>::Ty m_Sqrt(const Opnd0 &Op0) {
2206  return m_Intrinsic<Intrinsic::sqrt>(Op0);
2207}
2208 
2209template <typename Opnd0, typename Opnd1>
2210inline typename m_Intrinsic_Ty<Opnd0, Opnd1>::Ty m_CopySign(const Opnd0 &Op0,
2211                                                            const Opnd1 &Op1) {
2212  return m_Intrinsic<Intrinsic::copysign>(Op0, Op1);
2213}
2214 
2215template <typename Opnd0>
2216inline typename m_Intrinsic_Ty<Opnd0>::Ty m_VecReverse(const Opnd0 &Op0) {
2217  return m_Intrinsic<Intrinsic::experimental_vector_reverse>(Op0);
2218}
2219 
2220//===----------------------------------------------------------------------===//
2221// Matchers for two-operands operators with the operators in either order
2222//
2223 
2224/// Matches a BinaryOperator with LHS and RHS in either order.
2225template <typename LHS, typename RHS>
2226inline AnyBinaryOp_match<LHS, RHS, true> m_c_BinOp(const LHS &L, const RHS &R) {
2227  return AnyBinaryOp_match<LHS, RHS, true>(L, R);
2228}
2229 
2230/// Matches an ICmp with a predicate over LHS and RHS in either order.
2231/// Swaps the predicate if operands are commuted.
2232template <typename LHS, typename RHS>
2233inline CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>
2234m_c_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R) {
2235  return CmpClass_match<LHS, RHS, ICmpInst, ICmpInst::Predicate, true>(Pred, L,
2236                                                                       R);
2237}
2238 
2239/// Matches a specific opcode with LHS and RHS in either order.
2240template <typename LHS, typename RHS>
2241inline SpecificBinaryOp_match<LHS, RHS, true>
2242m_c_BinOp(unsigned Opcode, const LHS &L, const RHS &R) {
2243  return SpecificBinaryOp_match<LHS, RHS, true>(Opcode, L, R);
2244}
2245 
2246/// Matches a Add with LHS and RHS in either order.
2247template <typename LHS, typename RHS>
2248inline BinaryOp_match<LHS, RHS, Instruction::Add, true> m_c_Add(const LHS &L,
2249                                                                const RHS &R) {
2250  return BinaryOp_match<LHS, RHS, Instruction::Add, true>(L, R);
2251}
2252 
2253/// Matches a Mul with LHS and RHS in either order.
2254template <typename LHS, typename RHS>
2255inline BinaryOp_match<LHS, RHS, Instruction::Mul, true> m_c_Mul(const LHS &L,
2256                                                                const RHS &R) {
2257  return BinaryOp_match<LHS, RHS, Instruction::Mul, true>(L, R);
2258}
2259 
2260/// Matches an And with LHS and RHS in either order.
2261template <typename LHS, typename RHS>
2262inline BinaryOp_match<LHS, RHS, Instruction::And, true> m_c_And(const LHS &L,
2263                                                                const RHS &R) {
2264  return BinaryOp_match<LHS, RHS, Instruction::And, true>(L, R);
2265}
2266 
2267/// Matches an Or with LHS and RHS in either order.
2268template <typename LHS, typename RHS>
2269inline BinaryOp_match<LHS, RHS, Instruction::Or, true> m_c_Or(const LHS &L,
2270                                                              const RHS &R) {
2271  return BinaryOp_match<LHS, RHS, Instruction::Or, true>(L, R);
2272}
2273 
2274/// Matches an Xor with LHS and RHS in either order.
2275template <typename LHS, typename RHS>
2276inline BinaryOp_match<LHS, RHS, Instruction::Xor, true> m_c_Xor(const LHS &L,
2277                                                                const RHS &R) {
2278  return BinaryOp_match<LHS, RHS, Instruction::Xor, true>(L, R);
67
←
Returning without writing to 'R.VR'→
2279}
2280 
2281/// Matches a 'Neg' as 'sub 0, V'.
2282template <typename ValTy>
2283inline BinaryOp_match<cst_pred_ty<is_zero_int>, ValTy, Instruction::Sub>
2284m_Neg(const ValTy &V) {
2285  return m_Sub(m_ZeroInt(), V);
2286}
2287 
2288/// Matches a 'Neg' as 'sub nsw 0, V'.
2289template <typename ValTy>
2290inline OverflowingBinaryOp_match<cst_pred_ty<is_zero_int>, ValTy,
2291                                 Instruction::Sub,
2292                                 OverflowingBinaryOperator::NoSignedWrap>
2293m_NSWNeg(const ValTy &V) {
2294  return m_NSWSub(m_ZeroInt(), V);
2295}
2296 
2297/// Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
2298/// NOTE: we first match the 'Not' (by matching '-1'),
2299/// and only then match the inner matcher!
2300template <typename ValTy>
2301inline BinaryOp_match<cst_pred_ty<is_all_ones>, ValTy, Instruction::Xor, true>
2302m_Not(const ValTy &V) {
2303  return m_c_Xor(m_AllOnes(), V);
66
←
Calling 'm_c_Xor<llvm::PatternMatch::cstval_pred_ty<llvm::PatternMatch::is_all_ones, llvm::ConstantInt>, llvm::PatternMatch::bind_ty<llvm::Value>>'→
68
←
Returning from 'm_c_Xor<llvm::PatternMatch::cstval_pred_ty<llvm::PatternMatch::is_all_ones, llvm::ConstantInt>, llvm::PatternMatch::bind_ty<llvm::Value>>'→
69
←
Returning without writing to 'V.VR'→
2304}
2305 
2306template <typename ValTy> struct NotForbidUndef_match {
2307  ValTy Val;
2308  NotForbidUndef_match(const ValTy &V) : Val(V) {}
2309 
2310  template <typename OpTy> bool match(OpTy *V) {
2311    // We do not use m_c_Xor because that could match an arbitrary APInt that is
2312    // not -1 as C and then fail to match the other operand if it is -1.
2313    // This code should still work even when both operands are constants.
2314    Value *X;
2315    const APInt *C;
2316    if (m_Xor(m_Value(X), m_APIntForbidUndef(C)).match(V) && C->isAllOnes())
2317      return Val.match(X);
2318    if (m_Xor(m_APIntForbidUndef(C), m_Value(X)).match(V) && C->isAllOnes())
2319      return Val.match(X);
2320    return false;
2321  }
2322};
2323 
2324/// Matches a bitwise 'not' as 'xor V, -1' or 'xor -1, V'. For vectors, the
2325/// constant value must be composed of only -1 scalar elements.
2326template <typename ValTy>
2327inline NotForbidUndef_match<ValTy> m_NotForbidUndef(const ValTy &V) {
2328  return NotForbidUndef_match<ValTy>(V);
2329}
2330 
2331/// Matches an SMin with LHS and RHS in either order.
2332template <typename LHS, typename RHS>
2333inline MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>
2334m_c_SMin(const LHS &L, const RHS &R) {
2335  return MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>(L, R);
2336}
2337/// Matches an SMax with LHS and RHS in either order.
2338template <typename LHS, typename RHS>
2339inline MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>
2340m_c_SMax(const LHS &L, const RHS &R) {
2341  return MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>(L, R);
2342}
2343/// Matches a UMin with LHS and RHS in either order.
2344template <typename LHS, typename RHS>
2345inline MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>
2346m_c_UMin(const LHS &L, const RHS &R) {
2347  return MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>(L, R);
2348}
2349/// Matches a UMax with LHS and RHS in either order.
2350template <typename LHS, typename RHS>
2351inline MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>
2352m_c_UMax(const LHS &L, const RHS &R) {
2353  return MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>(L, R);
2354}
2355 
2356template <typename LHS, typename RHS>
2357inline match_combine_or<
2358    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, smax_pred_ty, true>,
2359                     MaxMin_match<ICmpInst, LHS, RHS, smin_pred_ty, true>>,
2360    match_combine_or<MaxMin_match<ICmpInst, LHS, RHS, umax_pred_ty, true>,
2361                     MaxMin_match<ICmpInst, LHS, RHS, umin_pred_ty, true>>>
2362m_c_MaxOrMin(const LHS &L, const RHS &R) {
2363  return m_CombineOr(m_CombineOr(m_c_SMax(L, R), m_c_SMin(L, R)),
2364                     m_CombineOr(m_c_UMax(L, R), m_c_UMin(L, R)));
2365}
2366 
2367template <Intrinsic::ID IntrID, typename T0, typename T1>
2368inline match_combine_or<typename m_Intrinsic_Ty<T0, T1>::Ty,
2369                        typename m_Intrinsic_Ty<T1, T0>::Ty>
2370m_c_Intrinsic(const T0 &Op0, const T1 &Op1) {
2371  return m_CombineOr(m_Intrinsic<IntrID>(Op0, Op1),
2372                     m_Intrinsic<IntrID>(Op1, Op0));
2373}
2374 
2375/// Matches FAdd with LHS and RHS in either order.
2376template <typename LHS, typename RHS>
2377inline BinaryOp_match<LHS, RHS, Instruction::FAdd, true>
2378m_c_FAdd(const LHS &L, const RHS &R) {
2379  return BinaryOp_match<LHS, RHS, Instruction::FAdd, true>(L, R);
2380}
2381 
2382/// Matches FMul with LHS and RHS in either order.
2383template <typename LHS, typename RHS>
2384inline BinaryOp_match<LHS, RHS, Instruction::FMul, true>
2385m_c_FMul(const LHS &L, const RHS &R) {
2386  return BinaryOp_match<LHS, RHS, Instruction::FMul, true>(L, R);
2387}
2388 
2389template <typename Opnd_t> struct Signum_match {
2390  Opnd_t Val;
2391  Signum_match(const Opnd_t &V) : Val(V) {}
2392 
2393  template <typename OpTy> bool match(OpTy *V) {
2394    unsigned TypeSize = V->getType()->getScalarSizeInBits();
2395    if (TypeSize == 0)
2396      return false;
2397 
2398    unsigned ShiftWidth = TypeSize - 1;
2399    Value *OpL = nullptr, *OpR = nullptr;
2400 
2401    // This is the representation of signum we match:
2402    //
2403    //  signum(x) == (x >> 63) | (-x >>u 63)
2404    //
2405    // An i1 value is its own signum, so it's correct to match
2406    //
2407    //  signum(x) == (x >> 0)  | (-x >>u 0)
2408    //
2409    // for i1 values.
2410 
2411    auto LHS = m_AShr(m_Value(OpL), m_SpecificInt(ShiftWidth));
2412    auto RHS = m_LShr(m_Neg(m_Value(OpR)), m_SpecificInt(ShiftWidth));
2413    auto Signum = m_Or(LHS, RHS);
2414 
2415    return Signum.match(V) && OpL == OpR && Val.match(OpL);
2416  }
2417};
2418 
2419/// Matches a signum pattern.
2420///
2421/// signum(x) =
2422///      x >  0  ->  1
2423///      x == 0  ->  0
2424///      x <  0  -> -1
2425template <typename Val_t> inline Signum_match<Val_t> m_Signum(const Val_t &V) {
2426  return Signum_match<Val_t>(V);
2427}
2428 
2429template <int Ind, typename Opnd_t> struct ExtractValue_match {
2430  Opnd_t Val;
2431  ExtractValue_match(const Opnd_t &V) : Val(V) {}
2432 
2433  template <typename OpTy> bool match(OpTy *V) {
2434    if (auto *I = dyn_cast<ExtractValueInst>(V)) {
2435      // If Ind is -1, don't inspect indices
2436      if (Ind != -1 &&
2437          !(I->getNumIndices() == 1 && I->getIndices()[0] == (unsigned)Ind))
2438        return false;
2439      return Val.match(I->getAggregateOperand());
2440    }
2441    return false;
2442  }
2443};
2444 
2445/// Match a single index ExtractValue instruction.
2446/// For example m_ExtractValue<1>(...)
2447template <int Ind, typename Val_t>
2448inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) {
2449  return ExtractValue_match<Ind, Val_t>(V);
2450}
2451 
2452/// Match an ExtractValue instruction with any index.
2453/// For example m_ExtractValue(...)
2454template <typename Val_t>
2455inline ExtractValue_match<-1, Val_t> m_ExtractValue(const Val_t &V) {
2456  return ExtractValue_match<-1, Val_t>(V);
2457}
2458 
2459/// Matcher for a single index InsertValue instruction.
2460template <int Ind, typename T0, typename T1> struct InsertValue_match {
2461  T0 Op0;
2462  T1 Op1;
2463 
2464  InsertValue_match(const T0 &Op0, const T1 &Op1) : Op0(Op0), Op1(Op1) {}
2465 
2466  template <typename OpTy> bool match(OpTy *V) {
2467    if (auto *I = dyn_cast<InsertValueInst>(V)) {
2468      return Op0.match(I->getOperand(0)) && Op1.match(I->getOperand(1)) &&
2469             I->getNumIndices() == 1 && Ind == I->getIndices()[0];
2470    }
2471    return false;
2472  }
2473};
2474 
2475/// Matches a single index InsertValue instruction.
2476template <int Ind, typename Val_t, typename Elt_t>
2477inline InsertValue_match<Ind, Val_t, Elt_t> m_InsertValue(const Val_t &Val,
2478                                                          const Elt_t &Elt) {
2479  return InsertValue_match<Ind, Val_t, Elt_t>(Val, Elt);
2480}
2481 
2482/// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or
2483/// the constant expression
2484///  `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>`
2485/// under the right conditions determined by DataLayout.
2486struct VScaleVal_match {
2487  template <typename ITy> bool match(ITy *V) {
2488    if (m_Intrinsic<Intrinsic::vscale>().match(V))
2489      return true;
2490 
2491    Value *Ptr;
2492    if (m_PtrToInt(m_Value(Ptr)).match(V)) {
2493      if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
2494        auto *DerefTy =
2495            dyn_cast<ScalableVectorType>(GEP->getSourceElementType());
2496        if (GEP->getNumIndices() == 1 && DerefTy &&
2497            DerefTy->getElementType()->isIntegerTy(8) &&
2498            m_Zero().match(GEP->getPointerOperand()) &&
2499            m_SpecificInt(1).match(GEP->idx_begin()->get()))
2500          return true;
2501      }
2502    }
2503 
2504    return false;
2505  }
2506};
2507 
2508inline VScaleVal_match m_VScale() {
2509  return VScaleVal_match();
2510}
2511 
2512template <typename LHS, typename RHS, unsigned Opcode, bool Commutable = false>
2513struct LogicalOp_match {
2514  LHS L;
2515  RHS R;
2516 
2517  LogicalOp_match(const LHS &L, const RHS &R) : L(L), R(R) {}
2518 
2519  template <typename T> bool match(T *V) {
2520    auto *I = dyn_cast<Instruction>(V);
2521    if (!I || !I->getType()->isIntOrIntVectorTy(1))
2522      return false;
2523 
2524    if (I->getOpcode() == Opcode) {
2525      auto *Op0 = I->getOperand(0);
2526      auto *Op1 = I->getOperand(1);
2527      return (L.match(Op0) && R.match(Op1)) ||
2528             (Commutable && L.match(Op1) && R.match(Op0));
2529    }
2530 
2531    if (auto *Select = dyn_cast<SelectInst>(I)) {
2532      auto *Cond = Select->getCondition();
2533      auto *TVal = Select->getTrueValue();
2534      auto *FVal = Select->getFalseValue();
2535 
2536      // Don't match a scalar select of bool vectors.
2537      // Transforms expect a single type for operands if this matches.
2538      if (Cond->getType() != Select->getType())
2539        return false;
2540 
2541      if (Opcode == Instruction::And) {
2542        auto *C = dyn_cast<Constant>(FVal);
2543        if (C && C->isNullValue())
2544          return (L.match(Cond) && R.match(TVal)) ||
2545                 (Commutable && L.match(TVal) && R.match(Cond));
2546      } else {
2547        assert(Opcode == Instruction::Or)(static_cast <bool> (Opcode == Instruction::Or) ? void (
0) : __assert_fail ("Opcode == Instruction::Or", "llvm/include/llvm/IR/PatternMatch.h"
, 2547, __extension__ __PRETTY_FUNCTION__));
2548        auto *C = dyn_cast<Constant>(TVal);
2549        if (C && C->isOneValue())
2550          return (L.match(Cond) && R.match(FVal)) ||
2551                 (Commutable && L.match(FVal) && R.match(Cond));
2552      }
2553    }
2554 
2555    return false;
2556  }
2557};
2558 
2559/// Matches L && R either in the form of L & R or L ? R : false.
2560/// Note that the latter form is poison-blocking.
2561template <typename LHS, typename RHS>
2562inline LogicalOp_match<LHS, RHS, Instruction::And> m_LogicalAnd(const LHS &L,
2563                                                                const RHS &R) {
2564  return LogicalOp_match<LHS, RHS, Instruction::And>(L, R);
2565}
2566 
2567/// Matches L && R where L and R are arbitrary values.
2568inline auto m_LogicalAnd() { return m_LogicalAnd(m_Value(), m_Value()); }
2569 
2570/// Matches L && R with LHS and RHS in either order.
2571template <typename LHS, typename RHS>
2572inline LogicalOp_match<LHS, RHS, Instruction::And, true>
2573m_c_LogicalAnd(const LHS &L, const RHS &R) {
2574  return LogicalOp_match<LHS, RHS, Instruction::And, true>(L, R);
2575}
2576 
2577/// Matches L || R either in the form of L | R or L ? true : R.
2578/// Note that the latter form is poison-blocking.
2579template <typename LHS, typename RHS>
2580inline LogicalOp_match<LHS, RHS, Instruction::Or> m_LogicalOr(const LHS &L,
2581                                                              const RHS &R) {
2582  return LogicalOp_match<LHS, RHS, Instruction::Or>(L, R);
2583}
2584 
2585/// Matches L || R where L and R are arbitrary values.
2586inline auto m_LogicalOr() { return m_LogicalOr(m_Value(), m_Value()); }
2587 
2588/// Matches L || R with LHS and RHS in either order.
2589template <typename LHS, typename RHS>
2590inline LogicalOp_match<LHS, RHS, Instruction::Or, true>
2591m_c_LogicalOr(const LHS &L, const RHS &R) {
2592  return LogicalOp_match<LHS, RHS, Instruction::Or, true>(L, R);
2593}
2594 
2595/// Matches either L && R or L || R,
2596/// either one being in the either binary or logical form.
2597/// Note that the latter form is poison-blocking.
2598template <typename LHS, typename RHS, bool Commutable = false>
2599inline auto m_LogicalOp(const LHS &L, const RHS &R) {
2600  return m_CombineOr(
2601      LogicalOp_match<LHS, RHS, Instruction::And, Commutable>(L, R),
2602      LogicalOp_match<LHS, RHS, Instruction::Or, Commutable>(L, R));
2603}
2604 
2605/// Matches either L && R or L || R where L and R are arbitrary values.
2606inline auto m_LogicalOp() { return m_LogicalOp(m_Value(), m_Value()); }
2607 
2608/// Matches either L && R or L || R with LHS and RHS in either order.
2609template <typename LHS, typename RHS>
2610inline auto m_c_LogicalOp(const LHS &L, const RHS &R) {
2611  return m_LogicalOp<LHS, RHS, /*Commutable=*/true>(L, R);
2612}
2613 
2614} // end namespace PatternMatch
2615} // end namespace llvm
2616 
2617#endif // LLVM_IR_PATTERNMATCH_H