|
LLVM 23.0.0git
|
#include "InstCombineInternal.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SmallVector.h"#include "llvm/Analysis/AssumptionCache.h"#include "llvm/Analysis/CmpInstAnalysis.h"#include "llvm/Analysis/InstructionSimplify.h"#include "llvm/Analysis/Loads.h"#include "llvm/Analysis/OverflowInstAnalysis.h"#include "llvm/Analysis/ValueTracking.h"#include "llvm/Analysis/VectorUtils.h"#include "llvm/IR/BasicBlock.h"#include "llvm/IR/Constant.h"#include "llvm/IR/ConstantRange.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/FMF.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/InstrTypes.h"#include "llvm/IR/Instruction.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/Operator.h"#include "llvm/IR/PatternMatch.h"#include "llvm/IR/ProfDataUtils.h"#include "llvm/IR/Type.h"#include "llvm/IR/User.h"#include "llvm/IR/Value.h"#include "llvm/Support/Casting.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/KnownBits.h"#include "llvm/Support/MathExtras.h"#include "llvm/Transforms/InstCombine/InstCombiner.h"#include <cassert>#include <optional>#include <utility>#include "llvm/Transforms/Utils/InstructionWorklist.h"Go to the source code of this file.
Namespaces | |
| namespace | llvm |
| This is an optimization pass for GlobalISel generic memory operations. | |
Macros | |
| #define | DEBUG_TYPE "instcombine" |
Functions | |
| static Instruction * | foldSelectBinOpIdentity (SelectInst &Sel, const TargetLibraryInfo &TLI, InstCombinerImpl &IC) |
| Replace a select operand based on an equality comparison with the identity constant of a binop. | |
| static Value * | foldSelectICmpAnd (SelectInst &Sel, Value *CondVal, Value *TrueVal, Value *FalseVal, Value *V, const APInt &AndMask, bool CreateAnd, InstCombiner::BuilderTy &Builder) |
| This folds: select (icmp eq (and X, C1)), TC, FC iff C1 is a power 2 and the difference between TC and FC is a power-of-2. | |
| static unsigned | getSelectFoldableOperands (BinaryOperator *I) |
| We want to turn code that looks like this: C = or A, B D = select cond, C, A into: C = select cond, B, 0 D = or A, C. | |
| static bool | isSelect01 (const APInt &C1I, const APInt &C2I) |
| static Value * | canoncalizeSelectICmpMinMax (const ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder, const SimplifyQuery &SQ) |
| static Value * | foldSelectICmpMinMax (const ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder, const SimplifyQuery &SQ) |
| Try to fold a select to a min/max intrinsic. | |
| static Instruction * | foldSelectICmpAndAnd (Type *SelType, const ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder) |
| We want to turn: (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1) into: zext (icmp ne i32 (and X, (or Y, (shl 1, Z))), 0) Note: Z may be 0 if lshr is missing. | |
| static Value * | foldSelectICmpAndZeroShl (const ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder) |
| We want to turn: (select (icmp eq (and X, C1), 0), 0, (shl [nsw/nuw] X, C2)); iff C1 is a mask and the number of its leading zeros is equal to C2 into: shl X, C2. | |
| static Value * | foldSelectICmpLshrAshr (const ICmpInst *IC, Value *TrueVal, Value *FalseVal, InstCombiner::BuilderTy &Builder) |
| We want to turn: (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1 (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0 into: ashr (X, Y) | |
| static Value * | foldSelectICmpAndBinOp (Value *CondVal, Value *TrueVal, Value *FalseVal, Value *V, const APInt &AndMask, bool CreateAnd, InstCombiner::BuilderTy &Builder) |
| We want to turn: (select (icmp eq (and X, C1), 0), Y, (BinOp Y, C2)) into: IF C2 u>= C1 (BinOp Y, (shl (and X, C1), C3)) ELSE (BinOp Y, (lshr (and X, C1), C3)) iff: 0 on the RHS is the identity value (i.e add, xor, shl, etc...) C1 and C2 are both powers of 2 where: IF C2 u>= C1 C3 = Log(C2) - Log(C1) ELSE C3 = Log(C1) - Log(C2) | |
| static Instruction * | foldSetClearBits (SelectInst &Sel, InstCombiner::BuilderTy &Builder) |
| Canonicalize a set or clear of a masked set of constant bits to select-of-constants form. | |
| static Instruction * | foldSelectZeroOrFixedOp (SelectInst &SI, InstCombinerImpl &IC) |
| static Value * | canonicalizeSaturatedSubtractUnsigned (const ICmpInst *ICI, const Value *TrueVal, const Value *FalseVal, InstCombiner::BuilderTy &Builder) |
| Transform patterns such as (a > b) ? | |
| static Value * | canonicalizeSaturatedSubtractSigned (const ICmpInst *ICI, const Value *TrueVal, const Value *FalseVal, InstCombiner::BuilderTy &Builder) |
| static Value * | canonicalizeSaturatedSubtract (const ICmpInst *ICI, const Value *TrueVal, const Value *FalseVal, InstCombiner::BuilderTy &Builder) |
| static Value * | canonicalizeSaturatedAddUnsigned (ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder) |
| static Value * | canonicalizeSaturatedAddSigned (ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder) |
| static Value * | canonicalizeSaturatedAdd (ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder) |
| static Value * | foldAbsDiff (ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder) |
| Try to match patterns with select and subtract as absolute difference. | |
| #define DEBUG_TYPE "instcombine" |
Definition at line 51 of file InstCombineSelect.cpp.
|
static |
Definition at line 661 of file InstCombineSelect.cpp.
References llvm::SimplifyQuery::AC, llvm::SimplifyQuery::CxtI, llvm::SimplifyQuery::DT, llvm::CmpInst::getInversePredicate(), llvm::CmpInst::getSwappedPredicate(), llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::isGuaranteedNotToBeUndef(), llvm::PatternMatch::m_NSWSub(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), llvm::SMin, and std::swap().
Referenced by foldSelectICmpMinMax().
|
static |
Definition at line 1409 of file InstCombineSelect.cpp.
References canonicalizeSaturatedAddSigned(), and canonicalizeSaturatedAddUnsigned().
|
static |
Definition at line 1290 of file InstCombineSelect.cpp.
References llvm::CallingConv::C, llvm::cast(), llvm::Constant::getAllOnesValue(), llvm::getFlippedStrictnessPredicateAndConstant(), llvm::CmpInst::getInversePredicate(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::isa(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isSignMask(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_MaxSignedValue(), llvm::PatternMatch::m_Negative(), llvm::PatternMatch::m_NSWSub(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_SignMask(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_SpecificIntAllowPoison(), llvm::PatternMatch::m_StrictlyPositive(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), std::swap(), and X.
Referenced by canonicalizeSaturatedAdd().
|
static |
Definition at line 1189 of file InstCombineSelect.cpp.
References llvm::CallingConv::C, llvm::cast(), llvm::CmpInst::getInversePredicate(), llvm::User::getOperand(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::getType(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_APIntAllowPoison(), llvm::PatternMatch::m_c_Add(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_NotForbidPoison(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificIntAllowPoison(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), std::swap(), X, and Y.
Referenced by canonicalizeSaturatedAdd().
|
static |
Definition at line 1173 of file InstCombineSelect.cpp.
References canonicalizeSaturatedSubtractSigned(), and canonicalizeSaturatedSubtractUnsigned().
|
static |
Definition at line 1147 of file InstCombineSelect.cpp.
References llvm::Constant::getNullValue(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::Value::getType(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::PatternMatch::m_MaxSignedValue(), llvm::MIPatternMatch::m_Neg(), llvm::PatternMatch::m_SignMask(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), and std::swap().
Referenced by canonicalizeSaturatedSubtract().
|
static |
Transform patterns such as (a > b) ?
a - b : 0 into usub.sat(a, b). There are 8 commuted/swapped variants of this pattern.
Definition at line 1080 of file InstCombineSelect.cpp.
References A(), assert(), B(), llvm::CallingConv::C, llvm::CmpInst::getInversePredicate(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::CmpInst::getSwappedPredicate(), llvm::Value::hasOneUse(), llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, llvm::CmpInst::isUnsigned(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), and std::swap().
Referenced by canonicalizeSaturatedSubtract().
|
static |
Try to match patterns with select and subtract as absolute difference.
Definition at line 1424 of file InstCombineSelect.cpp.
References A(), B(), llvm::dyn_cast(), llvm::CmpInst::getSwappedPredicate(), llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::MIPatternMatch::m_Neg(), llvm::PatternMatch::m_NSWSub(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::match(), and std::swap().
|
static |
Replace a select operand based on an equality comparison with the identity constant of a binop.
Definition at line 63 of file InstCombineSelect.cpp.
References llvm::CallingConv::C, llvm::cannotBeNegativeZero(), llvm::dyn_cast(), llvm::CmpInst::FCMP_OEQ, llvm::CmpInst::FCMP_UNE, llvm::InstCombinerImpl::fmulByZeroIsZero(), llvm::ConstantExpr::getBinOpIdentity(), llvm::SelectInst::getCondition(), llvm::Instruction::getFastMathFlags(), llvm::BinaryOperator::getOpcode(), llvm::User::getOperand(), llvm::InstCombiner::getSimplifyQuery(), llvm::Value::getType(), llvm::SimplifyQuery::getWithInstruction(), llvm::CmpInst::ICMP_EQ, llvm::Instruction::isCommutative(), llvm::ICmpInst::isEquality(), llvm::CmpInst::isFPPredicate(), llvm::PatternMatch::m_AnyZeroFP(), llvm::MIPatternMatch::m_BinOp(), llvm::PatternMatch::m_BinOp(), llvm::PatternMatch::m_c_BinOp(), llvm::PatternMatch::m_c_FMul(), llvm::PatternMatch::m_Cmp(), llvm::PatternMatch::m_Constant(), llvm::PatternMatch::m_FAbs(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), llvm::InstCombiner::replaceOperand(), X, and Y.
|
static |
This folds: select (icmp eq (and X, C1)), TC, FC iff C1 is a power 2 and the difference between TC and FC is a power-of-2.
To something like: (shr (and (X, C1)), (log2(C1) - log2(TC-FC))) + FC Or: (shl (and (X, C1)), (log2(TC-FC) - log2(C1))) + FC With some variations depending if FC is larger than TC, or the shift isn't needed, or the bit widths don't match.
Definition at line 153 of file InstCombineSelect.cpp.
References llvm::ConstantFoldBinaryOpOperands(), llvm::APInt::getBitWidth(), llvm::Instruction::getDataLayout(), llvm::Type::getScalarSizeInBits(), llvm::Value::getType(), llvm::Value::hasOneUse(), llvm::APInt::isPowerOf2(), llvm::APInt::isZero(), llvm::APInt::logBase2(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::match(), and Opc.
|
static |
We want to turn: (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1) into: zext (icmp ne i32 (and X, (or Y, (shl 1, Z))), 0) Note: Z may be 0 if lshr is missing.
Worst-case scenario is that we will replace 5 instructions with 5 different instructions, but we got rid of select.
Definition at line 763 of file InstCombineSelect.cpp.
References B(), llvm::Type::getScalarSizeInBits(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_ULT, llvm::PatternMatch::m_And(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_LShr(), llvm::PatternMatch::m_One(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt_ICMP(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), X, and Y.
|
static |
We want to turn: (select (icmp eq (and X, C1), 0), Y, (BinOp Y, C2)) into: IF C2 u>= C1 (BinOp Y, (shl (and X, C1), C3)) ELSE (BinOp Y, (lshr (and X, C1), C3)) iff: 0 on the RHS is the identity value (i.e add, xor, shl, etc...) C1 and C2 are both powers of 2 where: IF C2 u>= C1 C3 = Log(C2) - Log(C1) ELSE C3 = Log(C1) - Log(C2)
This transform handles cases where:
Definition at line 904 of file InstCombineSelect.cpp.
References llvm::cast(), llvm::dyn_cast(), llvm::ConstantExpr::getBinOpIdentity(), llvm::BinaryOperator::getOpcode(), llvm::Value::getType(), llvm::Value::hasOneUse(), llvm::APInt::logBase2(), llvm::PatternMatch::m_BinOp(), llvm::PatternMatch::m_Power2(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), and Y.
|
static |
We want to turn: (select (icmp eq (and X, C1), 0), 0, (shl [nsw/nuw] X, C2)); iff C1 is a mask and the number of its leading zeros is equal to C2 into: shl X, C2.
Definition at line 811 of file InstCombineSelect.cpp.
References llvm::APInt::countLeadingZeros(), llvm::dyn_cast(), llvm::APInt::getZExtValue(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::APInt::isMask(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), std::swap(), and X.
|
static |
We want to turn: (select (icmp sgt x, C), lshr (X, Y), ashr (X, Y)); iff C s>= -1 (select (icmp slt x, C), ashr (X, Y), lshr (X, Y)); iff C s>= 0 into: ashr (X, Y)
Definition at line 849 of file InstCombineSelect.cpp.
References llvm::cast(), llvm::APInt::getAllOnes(), llvm::Value::getName(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::Type::getScalarSizeInBits(), llvm::Value::getType(), llvm::APInt::getZero(), llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::Type::isIntOrIntVectorTy(), llvm::PatternMatch::m_AShr(), llvm::PatternMatch::m_LShr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt_ICMP(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), std::swap(), X, and Y.
|
static |
Try to fold a select to a min/max intrinsic.
Many cases are already handled by matchDecomposedSelectPattern but here we handle the cases where more extensive modification of the IR is required.
Definition at line 699 of file InstCombineSelect.cpp.
References canoncalizeSelectICmpMinMax(), llvm::cast(), llvm::CmpInst::getSwappedPredicate(), llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULT, llvm::isa(), llvm::isKnownNonZero(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_NSWAdd(), llvm::PatternMatch::m_NUWAdd(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::match(), and std::swap().
|
static |
Definition at line 1015 of file InstCombineSelect.cpp.
References llvm::cast(), llvm::dyn_cast(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::CmpInst::ICMP_NE, llvm::InstCombiner::InsertNewInstBefore(), llvm::isa(), llvm::ICmpInst::isEquality(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_c_Intrinsic(), llvm::PatternMatch::m_c_Mul(), llvm::PatternMatch::m_FShl(), llvm::PatternMatch::m_FShr(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_IDiv(), llvm::PatternMatch::m_IRem(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), llvm::Constant::mergeUndefsWith(), llvm::InstCombiner::replaceInstUsesWith(), llvm::InstCombiner::replaceOperand(), std::swap(), X, and Y.
|
static |
Canonicalize a set or clear of a masked set of constant bits to select-of-constants form.
Definition at line 972 of file InstCombineSelect.cpp.
References llvm::CallingConv::C, Cond, F, llvm::SelectInst::getCondition(), llvm::SelectInst::getFalseValue(), llvm::Constant::getNullValue(), llvm::SelectInst::getTrueValue(), llvm::Value::getType(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_APInt(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Or(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), T, and X.
|
static |
We want to turn code that looks like this: C = or A, B D = select cond, C, A into: C = select cond, B, 0 D = or A, C.
Assuming that the specified instruction is an operand to the select, return a bitmask indicating which operands of this instruction are foldable if they equal the other incoming value of the select.
Definition at line 251 of file InstCombineSelect.cpp.
References I.
Referenced by llvm::InstCombinerImpl::foldSelectIntoOp().
Definition at line 562 of file InstCombineSelect.cpp.
References llvm::APInt::isAllOnes(), llvm::APInt::isOne(), and llvm::APInt::isZero().
Referenced by llvm::InstCombinerImpl::foldSelectIntoOp().