45#include "llvm/IR/IntrinsicsAArch64.h"
46#include "llvm/IR/IntrinsicsAMDGPU.h"
47#include "llvm/IR/IntrinsicsARM.h"
48#include "llvm/IR/IntrinsicsHexagon.h"
77#define DEBUG_TYPE "instcombine"
81using namespace PatternMatch;
83STATISTIC(NumSimplified,
"Number of library calls simplified");
86 "instcombine-guard-widening-window",
88 cl::desc(
"How wide an instruction window to bypass looking for "
100 if (
IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
101 if (ITy->getBitWidth() < 32)
111 auto *Src =
MI->getRawSource();
112 while (isa<GetElementPtrInst>(Src) || isa<BitCastInst>(Src)) {
113 if (!Src->hasOneUse())
115 Src = cast<Instruction>(Src)->getOperand(0);
117 return isa<AllocaInst>(Src) && Src->hasOneUse();
123 if (!CopyDstAlign || *CopyDstAlign < DstAlign) {
124 MI->setDestAlignment(DstAlign);
130 if (!CopySrcAlign || *CopySrcAlign < SrcAlign) {
131 MI->setSourceAlignment(SrcAlign);
154 ConstantInt *MemOpLength = dyn_cast<ConstantInt>(
MI->getLength());
155 if (!MemOpLength)
return nullptr;
162 assert(
Size &&
"0-sized memory transferring should be removed already.");
171 if (isa<AtomicMemTransferInst>(
MI))
172 if (*CopyDstAlign <
Size || *CopySrcAlign <
Size)
177 cast<PointerType>(
MI->getArgOperand(1)->getType())->getAddressSpace();
179 cast<PointerType>(
MI->getArgOperand(0)->getType())->getAddressSpace();
188 if (
MDNode *M =
MI->getMetadata(LLVMContext::MD_tbaa)) {
190 }
else if (
MDNode *M =
MI->getMetadata(LLVMContext::MD_tbaa_struct)) {
191 if (M->getNumOperands() == 3 && M->getOperand(0) &&
192 mdconst::hasa<ConstantInt>(M->getOperand(0)) &&
193 mdconst::extract<ConstantInt>(M->getOperand(0))->isZero() &&
195 mdconst::hasa<ConstantInt>(M->getOperand(1)) &&
196 mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() ==
198 M->getOperand(2) && isa<MDNode>(M->getOperand(2)))
199 CopyMD = cast<MDNode>(M->getOperand(2));
209 MDNode *LoopMemParallelMD =
210 MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
211 if (LoopMemParallelMD)
212 L->
setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
213 MDNode *AccessGroupMD =
MI->getMetadata(LLVMContext::MD_access_group);
215 L->
setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
222 if (LoopMemParallelMD)
223 S->
setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
225 S->
setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
228 if (
auto *MT = dyn_cast<MemTransferInst>(
MI)) {
233 if (isa<AtomicMemTransferInst>(
MI)) {
245 const Align KnownAlignment =
248 if (!MemSetAlign || *MemSetAlign < KnownAlignment) {
249 MI->setDestAlignment(KnownAlignment);
265 if (isa<UndefValue>(
MI->getValue())) {
277 assert(Len &&
"0-sized memory setting should be removed already.");
278 const Align Alignment =
MI->getDestAlign().valueOrOne();
284 if (isa<AtomicMemSetInst>(
MI))
293 unsigned DstAddrSp = cast<PointerType>(Dest->
getType())->getAddressSpace();
303 if (
any_of(DAI->location_ops(), [&](
Value *V) { return V == FillC; }))
304 DAI->replaceVariableLocationOp(FillC, FillVal);
308 if (isa<AtomicMemSetInst>(
MI))
323 const Align Alignment =
341 LI->copyMetadata(II);
357 if (ConstMask->isNullValue())
361 if (ConstMask->isAllOnesValue()) {
370 if (isa<ScalableVectorType>(ConstMask->getType()))
397 if (ConstMask->isAllOnesValue())
399 auto *VecTy = cast<VectorType>(II.
getType());
400 const Align Alignment =
403 Alignment,
"load.scalar");
423 if (ConstMask->isNullValue())
432 new StoreInst(SplatValue, SplatPtr,
false, Alignment);
438 if (ConstMask->isAllOnesValue()) {
449 new StoreInst(Extract, SplatPtr,
false, Alignment);
454 if (isa<ScalableVectorType>(ConstMask->getType()))
481 auto *StrippedArg =
Arg->stripPointerCasts();
482 auto *StrippedInvariantGroupsArg = StrippedArg;
483 while (
auto *
Intr = dyn_cast<IntrinsicInst>(StrippedInvariantGroupsArg)) {
484 if (
Intr->getIntrinsicID() != Intrinsic::launder_invariant_group &&
485 Intr->getIntrinsicID() != Intrinsic::strip_invariant_group)
487 StrippedInvariantGroupsArg =
Intr->getArgOperand(0)->stripPointerCasts();
489 if (StrippedArg == StrippedInvariantGroupsArg)
492 Value *Result =
nullptr;
500 "simplifyInvariantGroupIntrinsic only handles launder and strip");
501 if (Result->getType()->getPointerAddressSpace() !=
504 if (Result->getType() != II.
getType())
507 return cast<Instruction>(Result);
513 "Expected cttz or ctlz intrinsic");
537 if (
auto *Sel = dyn_cast<SelectInst>(Op0))
586 if (PossibleZeros == DefiniteZeros) {
603 auto *
IT = dyn_cast<IntegerType>(Op0->
getType());
604 if (
IT &&
IT->getBitWidth() != 1 && !II.
getMetadata(LLVMContext::MD_range)) {
618 "Expected ctpop intrinsic");
661 if (
auto *Sel = dyn_cast<SelectInst>(Op0))
673 if ((~Known.
Zero).isPowerOf2())
674 return BinaryOperator::CreateLShr(
687 auto *
IT = dyn_cast<IntegerType>(Ty);
694 if (
IT->getBitWidth() != 1 && !II.
getMetadata(LLVMContext::MD_range)) {
717 auto *VecTy = cast<FixedVectorType>(II.
getType());
718 unsigned NumElts = VecTy->getNumElements();
721 if (!VecTy->getElementType()->isIntegerTy(8) || NumElts != 8)
726 for (
unsigned I = 0;
I < NumElts; ++
I) {
729 if (!COp || !isa<ConstantInt>(COp))
732 Indexes[
I] = cast<ConstantInt>(COp)->getLimitedValue();
735 if ((
unsigned)Indexes[
I] >= NumElts)
747 unsigned NumOperands) {
748 assert(
I.arg_size() >= NumOperands &&
"Not enough operands");
749 assert(
E.arg_size() >= NumOperands &&
"Not enough operands");
750 for (
unsigned i = 0; i < NumOperands; i++)
751 if (
I.getArgOperand(i) !=
E.getArgOperand(i))
772 for (; BI != BE; ++BI) {
773 if (
auto *
I = dyn_cast<IntrinsicInst>(&*BI)) {
774 if (
I->isDebugOrPseudoInst() ||
795 return I.getIntrinsicID() == Intrinsic::vastart ||
796 I.getIntrinsicID() == Intrinsic::vacopy;
802 assert(Call.arg_size() > 1 &&
"Need at least 2 args to swap");
803 Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1);
804 if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) {
805 Call.setArgOperand(0, Arg1);
806 Call.setArgOperand(1, Arg0);
823InstCombinerImpl::foldIntrinsicWithOverflowCommon(
IntrinsicInst *II) {
825 Value *OperationResult =
nullptr;
856 assert((MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin ||
857 MinMaxID == Intrinsic::umax || MinMaxID == Intrinsic::umin) &&
858 "Expected a min or max intrinsic");
863 const APInt *C0, *C1;
869 bool IsSigned = MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin;
870 auto *
Add = cast<BinaryOperator>(Op0);
871 if ((IsSigned && !
Add->hasNoSignedWrap()) ||
872 (!IsSigned && !
Add->hasNoUnsignedWrap()))
879 IsSigned ? C1->
ssub_ov(*C0, Overflow) : C1->
usub_ov(*C0, Overflow);
880 assert(!Overflow &&
"Expected simplify of min/max");
885 Value *NewMinMax =
Builder.CreateBinaryIntrinsic(MinMaxID,
X, NewMinMaxC);
886 return IsSigned ? BinaryOperator::CreateNSWAdd(NewMinMax,
Add->getOperand(1))
887 : BinaryOperator::CreateNUWAdd(NewMinMax,
Add->getOperand(1));
898 const APInt *MinValue, *MaxValue;
902 }
else if (
match(&MinMax1,
911 if (!(*MaxValue + 1).isPowerOf2() || -*MinValue != *MaxValue + 1)
914 unsigned NewBitWidth = (*MaxValue + 1).logBase2() + 1;
928 if (
AddSub->getOpcode() == Instruction::Add)
929 IntrinsicID = Intrinsic::sadd_sat;
930 else if (
AddSub->getOpcode() == Instruction::Sub)
931 IntrinsicID = Intrinsic::ssub_sat;
958 const APInt *C0, *C1;
964 case Intrinsic::smax:
968 case Intrinsic::smin:
972 case Intrinsic::umax:
976 case Intrinsic::umin:
997 if (!
LHS ||
LHS->getIntrinsicID() != MinMaxID)
1032 auto *InnerMM = dyn_cast<IntrinsicInst>(Inner);
1033 if (!InnerMM || InnerMM->getIntrinsicID() != MinMaxID ||
1051 if (!
LHS || !
RHS ||
LHS->getIntrinsicID() != MinMaxID ||
1052 RHS->getIntrinsicID() != MinMaxID ||
1062 Value *MinMaxOp =
nullptr;
1063 Value *ThirdOp =
nullptr;
1067 if (
D ==
A ||
C ==
A) {
1072 }
else if (
D ==
B ||
C ==
B) {
1081 if (
D ==
A ||
D ==
B) {
1086 }
else if (
C ==
A ||
C ==
B) {
1094 if (!MinMaxOp || !ThirdOp)
1111 case Intrinsic::smax:
1112 case Intrinsic::smin:
1113 case Intrinsic::umax:
1114 case Intrinsic::umin:
1115 case Intrinsic::fma:
1116 case Intrinsic::fshl:
1117 case Intrinsic::fshr:
1136 Type *SrcTy =
X->getType();
1137 for (
unsigned i = 1, e = II->
arg_size(); i != e; ++i) {
1140 X->getType() != SrcTy)
1146 Instruction *FPI = isa<FPMathOperator>(II) ? II :
nullptr;
1147 Value *NewIntrinsic =
1173 if (!II)
return visitCallBase(CI);
1177 if (
auto *AMI = dyn_cast<AtomicMemIntrinsic>(II))
1178 if (
ConstantInt *NumBytes = dyn_cast<ConstantInt>(AMI->getLength()))
1179 if (NumBytes->getSExtValue() < 0 ||
1180 (NumBytes->getZExtValue() % AMI->getElementSizeInBytes() != 0)) {
1182 assert(AMI->getType()->isVoidTy() &&
1183 "non void atomic unordered mem intrinsic");
1189 if (
auto *
MI = dyn_cast<AnyMemIntrinsic>(II)) {
1190 bool Changed =
false;
1193 if (
Constant *NumBytes = dyn_cast<Constant>(
MI->getLength())) {
1194 if (NumBytes->isNullValue())
1199 if (
auto *M = dyn_cast<MemIntrinsic>(
MI))
1200 if (M->isVolatile())
1206 if (
auto *MMI = dyn_cast<AnyMemMoveInst>(
MI)) {
1207 if (
GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))
1208 if (GVSrc->isConstant()) {
1211 isa<AtomicMemMoveInst>(MMI)
1212 ? Intrinsic::memcpy_element_unordered_atomic
1213 : Intrinsic::memcpy;
1224 if (MTI->getSource() == MTI->getDest())
1230 if (
auto *MTI = dyn_cast<AnyMemTransferInst>(
MI)) {
1233 }
else if (
auto *MSI = dyn_cast<AnyMemSetInst>(
MI)) {
1238 if (Changed)
return II;
1243 if (
auto *IIFVTy = dyn_cast<FixedVectorType>(II->
getType())) {
1244 auto VWidth = IIFVTy->getNumElements();
1245 APInt UndefElts(VWidth, 0);
1263 if (CI.
use_empty() && isa<ConstrainedFPIntrinsic>(CI)) {
1270 case Intrinsic::objectsize:
1274 case Intrinsic::abs: {
1276 bool IntMinIsPoison = cast<Constant>(II->
getArgOperand(1))->isOneValue();
1315 case Intrinsic::umin: {
1320 "Expected simplify of umin with max constant");
1327 case Intrinsic::umax: {
1331 (I0->
hasOneUse() || I1->hasOneUse()) &&
X->getType() ==
Y->getType()) {
1348 case Intrinsic::smax:
1349 case Intrinsic::smin: {
1353 (I0->
hasOneUse() || I1->hasOneUse()) &&
X->getType() ==
Y->getType()) {
1368 if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
1416 if (I0->
hasOneUse() && !I1->hasOneUse())
1428 if (IID == Intrinsic::smin || IID == Intrinsic::umax)
1440 if (
auto *Sel = dyn_cast<SelectInst>(I0))
1455 case Intrinsic::bitreverse: {
1459 X->getType()->isIntOrIntVectorTy(1)) {
1467 case Intrinsic::bswap: {
1484 cast<BinaryOperator>(IIOperand)->
getOpcode() == Instruction::Shl
1497 if (BW - LZ - TZ == 8) {
1498 assert(LZ != TZ &&
"active byte cannot be in the middle");
1500 return BinaryOperator::CreateNUWShl(
1503 return BinaryOperator::CreateExactLShr(
1509 unsigned C =
X->getType()->getScalarSizeInBits() - BW;
1516 case Intrinsic::masked_load:
1517 if (
Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II))
1520 case Intrinsic::masked_store:
1521 return simplifyMaskedStore(*II);
1522 case Intrinsic::masked_gather:
1523 return simplifyMaskedGather(*II);
1524 case Intrinsic::masked_scatter:
1525 return simplifyMaskedScatter(*II);
1526 case Intrinsic::launder_invariant_group:
1527 case Intrinsic::strip_invariant_group:
1531 case Intrinsic::powi:
1535 if (Power->isMinusOne())
1539 if (Power->equalsInt(2))
1543 if (!Power->getValue()[0]) {
1558 case Intrinsic::cttz:
1559 case Intrinsic::ctlz:
1564 case Intrinsic::ctpop:
1569 case Intrinsic::fshl:
1570 case Intrinsic::fshr: {
1582 if (ModuloC != ShAmtC)
1587 "Shift amount expected to be modulo bitwidth");
1592 if (IID == Intrinsic::fshr) {
1599 assert(IID == Intrinsic::fshl &&
1600 "All funnel shifts by simple constants should go left");
1605 return BinaryOperator::CreateShl(Op0, ShAmtC);
1610 return BinaryOperator::CreateLShr(Op1,
1636 case Intrinsic::uadd_with_overflow:
1637 case Intrinsic::sadd_with_overflow: {
1638 if (
Instruction *
I = foldIntrinsicWithOverflowCommon(II))
1645 const APInt *C0, *C1;
1648 bool IsSigned = IID == Intrinsic::sadd_with_overflow;
1654 IsSigned ? C1->
sadd_ov(*C0, Overflow) : C1->
uadd_ov(*C0, Overflow);
1663 case Intrinsic::umul_with_overflow:
1664 case Intrinsic::smul_with_overflow:
1665 case Intrinsic::usub_with_overflow:
1666 if (
Instruction *
I = foldIntrinsicWithOverflowCommon(II))
1670 case Intrinsic::ssub_with_overflow: {
1671 if (
Instruction *
I = foldIntrinsicWithOverflowCommon(II))
1693 case Intrinsic::uadd_sat:
1694 case Intrinsic::sadd_sat:
1695 case Intrinsic::usub_sat:
1696 case Intrinsic::ssub_sat: {
1698 Type *Ty =
SI->getType();
1728 C->isNotMinSignedValue()) {
1732 Intrinsic::sadd_sat, Arg0, NegVal));
1738 if (
auto *
Other = dyn_cast<IntrinsicInst>(Arg0)) {
1740 const APInt *Val, *Val2;
1743 IID == Intrinsic::uadd_sat || IID == Intrinsic::usub_sat;
1744 if (
Other->getIntrinsicID() == IID &&
1752 NewVal = Val->
sadd_ov(*Val2, Overflow);
1771 case Intrinsic::minnum:
1772 case Intrinsic::maxnum:
1773 case Intrinsic::minimum:
1774 case Intrinsic::maximum: {
1785 case Intrinsic::maxnum:
1786 NewIID = Intrinsic::minnum;
1788 case Intrinsic::minnum:
1789 NewIID = Intrinsic::maxnum;
1791 case Intrinsic::maximum:
1792 NewIID = Intrinsic::minimum;
1794 case Intrinsic::minimum:
1795 NewIID = Intrinsic::maximum;
1801 Instruction *FNeg = UnaryOperator::CreateFNeg(NewCall);
1808 if (
auto *M = dyn_cast<IntrinsicInst>(Arg0)) {
1816 case Intrinsic::maxnum:
1819 case Intrinsic::minnum:
1822 case Intrinsic::maximum:
1825 case Intrinsic::minimum:
1844 X->getType() ==
Y->getType()) {
1858 if (IID == Intrinsic::minimum || IID == Intrinsic::minnum)
1865 case Intrinsic::matrix_multiply: {
1879 Value *OpNotNeg, *NegatedOp;
1880 unsigned NegatedOpArg, OtherOpArg;
1915 NewArgs[NegatedOpArg] = OpNotNeg;
1922 case Intrinsic::fmuladd: {
1939 FAdd->copyFastMathFlags(II);
1945 case Intrinsic::fma: {
1970 FAdd->copyFastMathFlags(II);
1984 case Intrinsic::copysign: {
2015 case Intrinsic::fabs: {
2020 if (isa<Constant>(TVal) && isa<Constant>(FVal)) {
2033 Value *Magnitude, *Sign;
2045 case Intrinsic::ceil:
2046 case Intrinsic::floor:
2047 case Intrinsic::round:
2048 case Intrinsic::roundeven:
2049 case Intrinsic::nearbyint:
2050 case Intrinsic::rint:
2051 case Intrinsic::trunc: {
2060 case Intrinsic::cos:
2061 case Intrinsic::amdgcn_cos: {
2071 case Intrinsic::sin: {
2076 Instruction *FNeg = UnaryOperator::CreateFNeg(NewSin);
2082 case Intrinsic::ptrauth_auth:
2083 case Intrinsic::ptrauth_resign: {
2086 bool NeedSign = II->
getIntrinsicID() == Intrinsic::ptrauth_resign;
2092 Value *AuthKey =
nullptr, *AuthDisc =
nullptr, *BasePtr;
2109 if (AuthKey && NeedSign) {
2111 NewIntrin = Intrinsic::ptrauth_resign;
2112 }
else if (AuthKey) {
2114 NewIntrin = Intrinsic::ptrauth_auth;
2115 }
else if (NeedSign) {
2117 NewIntrin = Intrinsic::ptrauth_sign;
2140 case Intrinsic::arm_neon_vtbl1:
2141 case Intrinsic::aarch64_neon_tbl1:
2146 case Intrinsic::arm_neon_vmulls:
2147 case Intrinsic::arm_neon_vmullu:
2148 case Intrinsic::aarch64_neon_smull:
2149 case Intrinsic::aarch64_neon_umull: {
2154 if (isa<ConstantAggregateZero>(Arg0) || isa<ConstantAggregateZero>(Arg1)) {
2159 bool Zext = (IID == Intrinsic::arm_neon_vmullu ||
2160 IID == Intrinsic::aarch64_neon_umull);
2162 if (
Constant *CV0 = dyn_cast<Constant>(Arg0)) {
2163 if (
Constant *CV1 = dyn_cast<Constant>(Arg1)) {
2175 if (
Constant *CV1 = dyn_cast<Constant>(Arg1))
2177 dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))
2184 case Intrinsic::arm_neon_aesd:
2185 case Intrinsic::arm_neon_aese:
2186 case Intrinsic::aarch64_crypto_aesd:
2187 case Intrinsic::aarch64_crypto_aese: {
2201 case Intrinsic::hexagon_V6_vandvrt:
2202 case Intrinsic::hexagon_V6_vandvrt_128B: {
2204 if (
auto Op0 = dyn_cast<IntrinsicInst>(II->
getArgOperand(0))) {
2206 if (ID0 != Intrinsic::hexagon_V6_vandqrt &&
2207 ID0 != Intrinsic::hexagon_V6_vandqrt_128B)
2214 if ((
C & 0xFF) && (
C & 0xFF00) && (
C & 0xFF0000) && (
C & 0xFF000000))
2219 case Intrinsic::stackrestore: {
2220 enum class ClassifyResult {
2224 CallWithSideEffects,
2227 if (isa<AllocaInst>(
I))
2228 return ClassifyResult::Alloca;
2230 if (
auto *CI = dyn_cast<CallInst>(
I)) {
2231 if (
auto *II = dyn_cast<IntrinsicInst>(CI)) {
2233 return ClassifyResult::StackRestore;
2236 return ClassifyResult::CallWithSideEffects;
2239 return ClassifyResult::CallWithSideEffects;
2243 return ClassifyResult::None;
2250 if (SS->getIntrinsicID() == Intrinsic::stacksave &&
2253 bool CannotRemove =
false;
2254 for (++BI; &*BI != II; ++BI) {
2255 switch (Classify(&*BI)) {
2256 case ClassifyResult::None:
2260 case ClassifyResult::StackRestore:
2263 if (cast<IntrinsicInst>(*BI).getArgOperand(0) != SS)
2264 CannotRemove =
true;
2267 case ClassifyResult::Alloca:
2268 case ClassifyResult::CallWithSideEffects:
2271 CannotRemove =
true;
2287 bool CannotRemove =
false;
2288 for (++BI; &*BI != TI; ++BI) {
2289 switch (Classify(&*BI)) {
2290 case ClassifyResult::None:
2294 case ClassifyResult::StackRestore:
2298 case ClassifyResult::Alloca:
2299 case ClassifyResult::CallWithSideEffects:
2303 CannotRemove =
true;
2313 if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))
2317 case Intrinsic::lifetime_end:
2326 return I.getIntrinsicID() == Intrinsic::lifetime_start;
2330 case Intrinsic::assume: {
2339 assert(isa<AssumeInst>(Assume));
2349 if (
match(Next, m_Intrinsic<Intrinsic::assume>(
m_Specific(IIOperand))))
2350 return RemoveConditionFromAssume(Next);
2383 return RemoveConditionFromAssume(II);
2400 Replacement->insertBefore(Next);
2402 return RemoveConditionFromAssume(II);
2429 if (
auto *Replacement =
2432 Replacement->insertAfter(II);
2435 return RemoveConditionFromAssume(II);
2446 if (BOI.End - BOI.Begin > 2)
2457 if (BOI.End - BOI.Begin > 0) {
2464 if (BOI.End - BOI.Begin > 0)
2466 if (BOI.End - BOI.Begin > 1)
2487 case Intrinsic::experimental_guard: {
2498 Value *NextCond =
nullptr;
2500 m_Intrinsic<Intrinsic::experimental_guard>(
m_Value(NextCond)))) {
2505 if (CurrCond != NextCond) {
2507 while (MoveI != NextInst) {
2519 case Intrinsic::vector_insert: {
2523 auto *DstTy = dyn_cast<FixedVectorType>(II->
getType());
2524 auto *VecTy = dyn_cast<FixedVectorType>(Vec->
getType());
2525 auto *SubVecTy = dyn_cast<FixedVectorType>(SubVec->
getType());
2529 if (DstTy && VecTy && SubVecTy) {
2530 unsigned DstNumElts = DstTy->getNumElements();
2531 unsigned VecNumElts = VecTy->getNumElements();
2532 unsigned SubVecNumElts = SubVecTy->getNumElements();
2533 unsigned IdxN = cast<ConstantInt>(
Idx)->getZExtValue();
2536 if (VecNumElts == SubVecNumElts)
2545 for (i = 0; i != SubVecNumElts; ++i)
2547 for (; i != VecNumElts; ++i)
2553 for (
unsigned i = 0; i != IdxN; ++i)
2555 for (
unsigned i = DstNumElts; i != DstNumElts + SubVecNumElts; ++i)
2557 for (
unsigned i = IdxN + SubVecNumElts; i != DstNumElts; ++i)
2565 case Intrinsic::vector_extract: {
2572 unsigned ExtractIdx = cast<ConstantInt>(
Idx)->getZExtValue();
2573 Value *InsertTuple, *InsertIdx, *InsertValue;
2574 if (
match(Vec, m_Intrinsic<Intrinsic::vector_insert>(
m_Value(InsertTuple),
2577 InsertValue->
getType() == ReturnType) {
2578 unsigned Index = cast<ConstantInt>(InsertIdx)->getZExtValue();
2582 if (ExtractIdx ==
Index)
2593 auto *DstTy = dyn_cast<FixedVectorType>(ReturnType);
2594 auto *VecTy = dyn_cast<FixedVectorType>(Vec->
getType());
2598 if (DstTy && VecTy) {
2599 unsigned DstNumElts = DstTy->getNumElements();
2600 unsigned VecNumElts = VecTy->getNumElements();
2601 unsigned IdxN = cast<ConstantInt>(
Idx)->getZExtValue();
2604 if (VecNumElts == DstNumElts) {
2610 for (
unsigned i = 0; i != DstNumElts; ++i)
2611 Mask.push_back(IdxN + i);
2618 case Intrinsic::experimental_vector_reverse: {
2622 auto *OldBinOp = cast<BinaryOperator>(Vec);
2628 OldBinOp->getOpcode(),
X,
Y, OldBinOp,
2629 OldBinOp->getName(), II));
2634 OldBinOp->getOpcode(),
X, BO1,
2635 OldBinOp, OldBinOp->
getName(), II));
2640 OldBinOp->getOpcode(), BO0,
Y,
2641 OldBinOp, OldBinOp->getName(), II));
2645 auto *OldUnOp = cast<UnaryOperator>(Vec);
2647 OldUnOp->getOpcode(),
X, OldUnOp, OldUnOp->getName(), II);
2652 case Intrinsic::vector_reduce_or:
2653 case Intrinsic::vector_reduce_and: {
2664 if (
auto *FTy = dyn_cast<FixedVectorType>(Vect->
getType()))
2668 if (IID == Intrinsic::vector_reduce_and) {
2672 assert(IID == Intrinsic::vector_reduce_or &&
2673 "Expected or reduction.");
2684 case Intrinsic::vector_reduce_add: {
2685 if (IID == Intrinsic::vector_reduce_add) {
2695 if (
auto *FTy = dyn_cast<FixedVectorType>(Vect->
getType()))
2703 cast<Instruction>(
Arg)->
getOpcode() == Instruction::SExt)
2711 case Intrinsic::vector_reduce_xor: {
2712 if (IID == Intrinsic::vector_reduce_xor) {
2723 if (
auto *FTy = dyn_cast<FixedVectorType>(Vect->
getType()))
2735 case Intrinsic::vector_reduce_mul: {
2736 if (IID == Intrinsic::vector_reduce_mul) {
2746 if (
auto *FTy = dyn_cast<FixedVectorType>(Vect->
getType()))
2757 case Intrinsic::vector_reduce_umin:
2758 case Intrinsic::vector_reduce_umax: {
2759 if (IID == Intrinsic::vector_reduce_umin ||
2760 IID == Intrinsic::vector_reduce_umax) {
2770 if (
auto *FTy = dyn_cast<FixedVectorType>(Vect->
getType()))
2772 Value *Res = IID == Intrinsic::vector_reduce_umin
2784 case Intrinsic::vector_reduce_smin:
2785 case Intrinsic::vector_reduce_smax: {
2786 if (IID == Intrinsic::vector_reduce_smin ||
2787 IID == Intrinsic::vector_reduce_smax) {
2805 if (
auto *FTy = dyn_cast<FixedVectorType>(Vect->
getType()))
2809 ExtOpc = cast<CastInst>(
Arg)->getOpcode();
2810 Value *Res = ((IID == Intrinsic::vector_reduce_smin) ==
2811 (ExtOpc == Instruction::CastOps::ZExt))
2822 case Intrinsic::vector_reduce_fmax:
2823 case Intrinsic::vector_reduce_fmin:
2824 case Intrinsic::vector_reduce_fadd:
2825 case Intrinsic::vector_reduce_fmul: {
2826 bool CanBeReassociated = (IID != Intrinsic::vector_reduce_fadd &&
2827 IID != Intrinsic::vector_reduce_fmul) ||
2829 const unsigned ArgIdx = (IID == Intrinsic::vector_reduce_fadd ||
2830 IID == Intrinsic::vector_reduce_fmul)
2836 if (!isa<FixedVectorType>(
Arg->getType()) || !CanBeReassociated ||
2838 !cast<ShuffleVectorInst>(
Arg)->isSingleSource())
2840 int Sz = Mask.size();
2842 for (
int Idx : Mask) {
2849 if (UsedIndices.
all()) {
2869 return visitCallBase(*II);
2884 if (FI1SyncScope != FI2->getSyncScopeID() ||
2891 if (NFI && isIdenticalOrStrongerFence(NFI, &FI))
2895 if (isIdenticalOrStrongerFence(PFI, &FI))
2902 return visitCallBase(II);
2907 return visitCallBase(CBI);
2924 if (isa<GCStatepointInst>(Call) || isa<GCRelocateInst>(Call) ||
2925 isa<GCResultInst>(Call))
2930 if (SrcTy->isOpaque())
2936 if (!Call.isPassPointeeByValueArgument(ix))
2941 if (!Call.isByValArgument(ix))
2944 Type *SrcElemTy = SrcTy->getNonOpaquePointerElementType();
2945 Type *DstElemTy = Call.getParamByValType(ix);
2948 if (
DL.getTypeAllocSize(SrcElemTy) !=
DL.getTypeAllocSize(DstElemTy))
2970 if (
Value *With = Simplifier.optimizeCall(CI,
Builder)) {
2982 if (Underlying != TrampMem &&
2983 (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))
2985 if (!isa<AllocaInst>(Underlying))
2997 InitTrampoline = II;
3007 if (!InitTrampoline)
3011 if (InitTrampoline->
getOperand(0) != TrampMem)
3014 return InitTrampoline;
3054bool InstCombinerImpl::annotateAnyAllocSite(
CallBase &Call,
3060 bool Changed =
false;
3062 if (!
Call.getType()->isPointerTy())
3069 if (
Call.hasRetAttr(Attribute::NonNull)) {
3070 Changed = !
Call.hasRetAttr(Attribute::Dereferenceable);
3072 Call.getContext(),
Size->getLimitedValue()));
3074 Changed = !
Call.hasRetAttr(Attribute::DereferenceableOrNull);
3076 Call.getContext(),
Size->getLimitedValue()));
3085 ConstantInt *AlignOpC = dyn_cast<ConstantInt>(Alignment);
3089 Align ExistingAlign =
Call.getRetAlign().valueOrOne();
3091 if (NewAlign > ExistingAlign) {
3103 bool Changed = annotateAnyAllocSite(Call, &
TLI);
3112 if (V->getType()->isPointerTy() &&
3113 !
Call.paramHasAttr(ArgNo, Attribute::NonNull) &&
3119 assert(ArgNo ==
Call.arg_size() &&
"Call arguments not processed correctly.");
3121 if (!ArgNos.
empty()) {
3126 Call.setAttributes(AS);
3135 transformConstExprCastCall(Call))
3142 LLVM_DEBUG(
dbgs() <<
"Removing convergent attr from instr " << Call
3144 Call.setNotConvergent();
3166 if (isa<CallInst>(OldCall))
3171 cast<CallBase>(OldCall)->setCalledFunction(
3180 if ((isa<ConstantPointerNull>(
Callee) &&
3182 isa<UndefValue>(
Callee)) {
3185 if (!
Call.getType()->isVoidTy())
3188 if (
Call.isTerminator()) {
3199 return transformCallThroughTrampoline(Call, *II);
3203 if (FTy->isVarArg()) {
3204 int ix = FTy->getNumParams();
3207 for (
auto I =
Call.arg_begin() + FTy->getNumParams(),
E =
Call.arg_end();
3208 I !=
E; ++
I, ++ix) {
3216 if (!NewTy->isOpaque() &&
Call.isByValArgument(ix)) {
3217 Call.removeParamAttr(ix, Attribute::ByVal);
3220 NewTy->getNonOpaquePointerElementType()));
3227 if (isa<InlineAsm>(
Callee) && !
Call.doesNotThrow()) {
3229 if (!
IA->canThrow()) {
3232 Call.setDoesNotThrow();
3240 if (
CallInst *CI = dyn_cast<CallInst>(&Call)) {
3247 if (!
Call.use_empty() && !
Call.isMustTailCall())
3248 if (
Value *ReturnedArg =
Call.getReturnedArgOperand()) {
3250 Type *RetArgTy = ReturnedArg->getType();
3259 if (Bundle && !
Call.isIndirectCall()) {
3263 ConstantInt *ExpectedType = cast<ConstantInt>(Bundle->Inputs[0]);
3266 FunctionType = mdconst::extract<ConstantInt>(MD->getOperand(0));
3270 dbgs() <<
Call.getModule()->getName()
3271 <<
": warning: kcfi: " <<
Call.getCaller()->getName()
3272 <<
": call to " << CalleeF->
getName()
3273 <<
" using a mismatching function pointer type\n";
3284 switch (
Call.getIntrinsicID()) {
3285 case Intrinsic::experimental_gc_statepoint: {
3301 if (isa<UndefValue>(DerivedPtr) || isa<UndefValue>(BasePtr)) {
3307 if (
auto *PT = dyn_cast<PointerType>(GCR.
getType())) {
3311 if (isa<ConstantPointerNull>(DerivedPtr)) {
3339 LiveGcValues.
insert(BasePtr);
3340 LiveGcValues.
insert(DerivedPtr);
3342 std::optional<OperandBundleUse> Bundle =
3344 unsigned NumOfGCLives = LiveGcValues.
size();
3345 if (!Bundle || NumOfGCLives == Bundle->Inputs.size())
3349 std::vector<Value *> NewLiveGc;
3350 for (
Value *V : Bundle->Inputs) {
3351 if (Val2Idx.
count(V))
3353 if (LiveGcValues.
count(V)) {
3354 Val2Idx[V] = NewLiveGc.
size();
3355 NewLiveGc.push_back(V);
3357 Val2Idx[V] = NumOfGCLives;
3363 assert(Val2Idx.
count(BasePtr) && Val2Idx[BasePtr] != NumOfGCLives &&
3364 "Missed live gc for base pointer");
3368 assert(Val2Idx.
count(DerivedPtr) && Val2Idx[DerivedPtr] != NumOfGCLives &&
3369 "Missed live gc for derived pointer");
3380 return Changed ? &
Call :
nullptr;
3385bool InstCombinerImpl::transformConstExprCastCall(
CallBase &Call) {
3387 dyn_cast<Function>(
Call.getCalledOperand()->stripPointerCasts());
3394 if (
Callee->hasFnAttribute(
"thunk"))
3401 if (
Call.isMustTailCall())
3412 Type *NewRetTy = FT->getReturnType();
3415 if (OldRetTy != NewRetTy) {
3421 if (
Callee->isDeclaration())
3424 if (!
Caller->use_empty() &&
3440 if (!
Caller->use_empty()) {
3442 if (
auto *II = dyn_cast<InvokeInst>(Caller))
3443 PhisNotSupportedBlock = II->getNormalDest();
3444 if (
auto *CB = dyn_cast<CallBrInst>(Caller))
3445 PhisNotSupportedBlock = CB->getDefaultDest();
3446 if (PhisNotSupportedBlock)
3448 if (
PHINode *PN = dyn_cast<PHINode>(U))
3449 if (PN->getParent() == PhisNotSupportedBlock)
3454 unsigned NumActualArgs =
Call.arg_size();
3455 unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
3465 if (
Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||
3466 Callee->getAttributes().hasAttrSomewhere(Attribute::Preallocated))
3469 auto AI =
Call.arg_begin();
3470 for (
unsigned i = 0, e = NumCommonArgs; i !=
e; ++i, ++AI) {
3471 Type *ParamTy = FT->getParamType(i);
3472 Type *ActTy = (*AI)->getType();
3483 if (
Call.isInAllocaArgument(i) ||
3491 Callee->getAttributes().hasParamAttr(i, Attribute::ByVal))
3496 if (ParamTy != ActTy && CallerPAL.
hasParamAttr(i, Attribute::ByVal)) {
3497 PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
3501 if (!ParamPTy->isOpaque()) {
3502 Type *ParamElTy = ParamPTy->getNonOpaquePointerElementType();
3506 Type *CurElTy =
Call.getParamByValType(i);
3513 if (
Callee->isDeclaration()) {
3515 if (FT->getNumParams() < NumActualArgs && !FT->isVarArg())
3521 if (FT->isVarArg() !=
Call.getFunctionType()->isVarArg())
3527 if (FT->isVarArg() &&
Call.getFunctionType()->isVarArg() &&
3528 FT->getNumParams() !=
Call.getFunctionType()->getNumParams())
3532 if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&
3547 Args.reserve(NumActualArgs);
3548 ArgAttrs.
reserve(NumActualArgs);
3558 AI =
Call.arg_begin();
3559 for (
unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
3560 Type *ParamTy = FT->getParamType(i);
3562 Value *NewArg = *AI;
3563 if ((*AI)->getType() != ParamTy)
3565 Args.push_back(NewArg);
3574 Ctx, IncompatibleAttrs));
3585 for (
unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) {
3591 if (FT->getNumParams() < NumActualArgs) {
3593 if (FT->isVarArg()) {
3595 for (
unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
3597 Value *NewArg = *AI;
3598 if (PTy != (*AI)->getType()) {
3604 Args.push_back(NewArg);
3617 assert((ArgAttrs.
size() == FT->getNumParams() || FT->isVarArg()) &&
3618 "missing argument attributes");
3623 Call.getOperandBundlesAsDefs(OpBundles);
3626 if (
InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
3628 II->getUnwindDest(), Args, OpBundles);
3629 }
else if (
CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
3631 CBI->getIndirectDests(), Args, OpBundles);
3635 cast<CallInst>(Caller)->getTailCallKind());
3642 NewCall->
copyMetadata(*Caller, {LLVMContext::MD_prof});
3647 if (OldRetTy !=
NV->getType() && !
Caller->use_empty()) {
3648 if (!
NV->getType()->isVoidTy()) {
3650 NC->setDebugLoc(
Caller->getDebugLoc());
3653 assert(InsertPt &&
"No place to insert cast");
3661 if (!
Caller->use_empty())
3663 else if (
Caller->hasValueHandle()) {
3664 if (OldRetTy ==
NV->getType())
3679InstCombinerImpl::transformCallThroughTrampoline(
CallBase &Call,
3688 if (
Attrs.hasAttrSomewhere(Attribute::Nest))
3696 unsigned NestArgNo = 0;
3697 Type *NestTy =
nullptr;
3702 E = NestFTy->param_end();
3703 I !=
E; ++NestArgNo, ++
I) {
3714 std::vector<Value*> NewArgs;
3715 std::vector<AttributeSet> NewArgAttrs;
3716 NewArgs.reserve(
Call.arg_size() + 1);
3717 NewArgAttrs.reserve(
Call.arg_size());
3724 auto I =
Call.arg_begin(),
E =
Call.arg_end();
3726 if (ArgNo == NestArgNo) {
3729 if (NestVal->
getType() != NestTy)
3731 NewArgs.push_back(NestVal);
3732 NewArgAttrs.push_back(NestAttr);
3739 NewArgs.push_back(*
I);
3740 NewArgAttrs.push_back(
Attrs.getParamAttrs(ArgNo));
3751 std::vector<Type*> NewTypes;
3752 NewTypes.reserve(FTy->getNumParams()+1);
3759 E = FTy->param_end();
3762 if (ArgNo == NestArgNo)
3764 NewTypes.push_back(NestTy);
3770 NewTypes.push_back(*
I);
3787 Attrs.getRetAttrs(), NewArgAttrs);
3790 Call.getOperandBundlesAsDefs(OpBundles);
3793 if (
InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {
3795 II->getNormalDest(), II->getUnwindDest(),
3796 NewArgs, OpBundles);
3797 cast<InvokeInst>(NewCaller)->setCallingConv(II->
getCallingConv());
3798 cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
3799 }
else if (
CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {
3802 CBI->getIndirectDests(), NewArgs, OpBundles);
3803 cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());
3804 cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);
3807 cast<CallInst>(NewCaller)->setTailCallKind(
3808 cast<CallInst>(Call).getTailCallKind());
3809 cast<CallInst>(NewCaller)->setCallingConv(
3810 cast<CallInst>(Call).getCallingConv());
3811 cast<CallInst>(NewCaller)->setAttributes(NewPAL);
3823 Call.setCalledFunction(FTy, NewCallee);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static Type * getPromotedType(Type *Ty)
Return the specified type promoted as it would be to pass though a va_arg area.
static Instruction * createOverflowTuple(IntrinsicInst *II, Value *Result, Constant *Overflow)
Creates a result tuple for an overflow intrinsic II with a given Result and a constant Overflow value...
static bool isSafeToEliminateVarargsCast(const CallBase &Call, const DataLayout &DL, const CastInst *const CI, const int ix)
If this cast does not affect the value passed through the varargs area, we can eliminate the use of t...
static IntrinsicInst * findInitTrampolineFromAlloca(Value *TrampMem)
static bool removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC, std::function< bool(const IntrinsicInst &)> IsStart)
static Instruction * reassociateMinMaxWithConstantInOperand(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If this min/max has a matching min/max operand with a constant, try to push the constant operand into...
static Instruction * moveAddAfterMinMax(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0.
static Instruction * simplifyInvariantGroupIntrinsic(IntrinsicInst &II, InstCombinerImpl &IC)
This function transforms launder.invariant.group and strip.invariant.group like: launder(launder(x)) ...
static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E, unsigned NumOperands)
static cl::opt< unsigned > GuardWideningWindow("instcombine-guard-widening-window", cl::init(3), cl::desc("How wide an instruction window to bypass looking for " "another guard"))
static bool hasUndefSource(AnyMemTransferInst *MI)
Recognize a memcpy/memmove from a trivially otherwise unused alloca.
static Instruction * foldShuffledIntrinsicOperands(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If all arguments of the intrinsic are unary shuffles with the same mask, try to shuffle after the int...
static Instruction * factorizeMinMaxTree(IntrinsicInst *II)
Reduce a sequence of min/max intrinsics with a common operand.
static Value * simplifyNeonTbl1(const IntrinsicInst &II, InstCombiner::BuilderTy &Builder)
Convert a table lookup to shufflevector if the mask is constant.
static Instruction * foldClampRangeOfTwo(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If we have a clamp pattern like max (min X, 42), 41 – where the output can only be one of two possibl...
static IntrinsicInst * findInitTrampolineFromBB(IntrinsicInst *AdjustTramp, Value *TrampMem)
static Instruction * reassociateMinMaxWithConstants(IntrinsicInst *II)
If this min/max has a constant operand and an operand that is a matching min/max with a constant oper...
static Instruction * foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC)
static Instruction * foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC)
static IntrinsicInst * findInitTrampoline(Value *Callee)
static std::optional< bool > getKnownSign(Value *Op, Instruction *CxtI, const DataLayout &DL, AssumptionCache *AC, DominatorTree *DT)
static CallInst * canonicalizeConstantArg0ToArg1(CallInst &Call)
This file provides internal interfaces used to implement the InstCombine.
This file provides the interface for the instcombine pass implementation.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the SmallBitVector class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, bool IgnoreLocals=false)
Returns a bitmask that should be unconditionally applied to the ModRef info of a memory location.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
APInt usub_ov(const APInt &RHS, bool &Overflow) const
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
APInt sadd_ov(const APInt &RHS, bool &Overflow) const
APInt uadd_ov(const APInt &RHS, bool &Overflow) const
APInt uadd_sat(const APInt &RHS) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
APInt ssub_ov(const APInt &RHS, bool &Overflow) const
static APSInt getMinValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the minimum integer value with the given bit width and signedness.
static APSInt getMaxValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the maximum integer value with the given bit width and signedness.
This class represents any memset intrinsic.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A cache of @llvm.assume calls within a function.
void updateAffectedValues(CondGuardInst *CI)
Update the cache of values being affected by this assumption (i.e.
void registerAssumption(CondGuardInst *CI)
Add an @llvm.assume intrinsic to this function's cache.
bool overlaps(const AttributeMask &AM) const
Return true if the builder has any attribute that's in the specified builder.
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
bool isEmpty() const
Return true if there are no attributes.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasAttrSomewhere(Attribute::AttrKind Kind, unsigned *Index=nullptr) const
Return true if the specified attribute is set for at least one parameter or for the return value.
bool hasParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Return true if the attribute exists for the given argument.
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeList addParamAttribute(LLVMContext &C, unsigned ArgNo, Attribute::AttrKind Kind) const
Add an argument attribute to the list.
bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
AttributeSet removeAttributes(LLVMContext &C, const AttributeMask &AttrsToRemove) const
Remove the specified attributes from this set.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes)
static Attribute getWithByValType(LLVMContext &Context, Type *Ty)
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
InstListType::reverse_iterator reverse_iterator
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
bool isSigned() const
Whether the intrinsic is signed or unsigned.
Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
static BinaryOperator * CreateFDivFMF(Value *V1, Value *V2, Instruction *FMFSource, const Twine &Name="")
static BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
static BinaryOperator * CreateNSW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
static BinaryOperator * CreateNot(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
static BinaryOperator * CreateNSWNeg(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
static BinaryOperator * CreateFMulFMF(Value *V1, Value *V2, Instruction *FMFSource, const Twine &Name="")
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
bundle_op_iterator bundle_op_info_begin()
Return the start of the list of BundleOpInfo instances associated with this OperandBundleUser.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
unsigned getNumOperandBundles() const
Return the number of operand bundles associated with this User.
CallingConv::ID getCallingConv() const
static CallBase * Create(CallBase *CB, ArrayRef< OperandBundleDef > Bundles, Instruction *InsertPt=nullptr)
Create a clone of CB with a different set of operand bundles and insert it before InsertPt.
static CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, Instruction *InsertPt=nullptr)
Create a clone of CB with operand bundle ID removed.
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
bool doesNotThrow() const
Determine if the call cannot unwind.
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
bool hasOperandBundles() const
Return true if this User has any operand bundles.
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
static CallBrInst * Create(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, ArrayRef< BasicBlock * > IndirectDests, ArrayRef< Value * > Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
This class represents a function call, abstracting a target machine's calling convention.
bool isNoTailCall() const
void setTailCallKind(TailCallKind TCK)
bool isMustTailCall() const
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
static Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
static CastInst * CreateIntegerCast(Value *S, Type *Ty, bool isSigned, const Twine &Name="", Instruction *InsertBefore=nullptr)
Create a ZExt, BitCast, or Trunc for int -> int casts.
static CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
bool isLosslessCast() const
A lossless cast is one that does not alter the basic value.
static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
static ConstantAggregateZero * get(Type *Ty)
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getZExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getICmp(unsigned short pred, Constant *LHS, Constant *RHS, bool OnlyIfReduced=false)
get* - Return some common constants without having to specify the full Instruction::OPCODE identifier...
static Constant * getSelect(Constant *C, Constant *V1, Constant *V2, Type *OnlyIfReducedTy=nullptr)
Select constant expr.
static Constant * getIntegerCast(Constant *C, Type *Ty, bool IsSigned)
Create a ZExt, Bitcast or Trunc for integer -> integer casts.
static Constant * getSExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getMul(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getNeg(Constant *C, bool HasNUW=false, bool HasNSW=false)
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
This is the shared class of boolean and integer constants.
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
static ConstantInt * getTrue(LLVMContext &Context)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
static ConstantInt * getBool(LLVMContext &Context, bool V)
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
This class represents an extension of floating point types.
bool noSignedZeros() const
An instruction for ordering other memory operations.
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this fence instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this fence instruction.
FunctionType * getFunctionType()
Class to represent function types.
Type::subtype_iterator param_iterator
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
bool isConvergent() const
Determine if the call is convergent.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
bool doesNotThrow() const
Determine if the function cannot unwind.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Represents calls to the gc.relocate intrinsic.
Value * getBasePtr() const
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
Value * getDerivedPtr() const
unsigned getDerivedPtrIndex() const
The index into the associate statepoint's argument list which contains the pointer whose relocation t...
Represents a gc.statepoint intrinsic call.
std::vector< const GCRelocateInst * > getGCRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
PointerType * getType() const
Global values are always pointers.
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
CallBrInst * CreateCallBr(FunctionType *Ty, Value *Callee, BasicBlock *DefaultDest, ArrayRef< BasicBlock * > IndirectDests, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="")
Create a callbr instruction.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateVScale(Constant *Scaling, const Twine &Name="")
Create a call to llvm.vscale, multiplied by Scaling.
Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateFNegFMF(Value *V, Instruction *FMFSource, const Twine &Name="")
Copy fast-math-flags from an instruction rather than using the builder's default FMF.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
InvokeInst * CreateInvoke(FunctionType *Ty, Value *Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > OpBundles, const Twine &Name="")
Create an invoke instruction.
CallInst * CreateAddReduce(Value *Src)
Create a vector int add reduction intrinsic of the source vector.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
CallInst * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
bool SimplifyDemandedBits(Instruction *I, unsigned Op, const APInt &DemandedMask, KnownBits &Known, unsigned Depth=0) override
This form of SimplifyDemandedBits simplifies the specified instruction operand if possible,...
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
Instruction * SimplifyAnyMemSet(AnyMemSetInst *MI)
Instruction * visitFree(CallInst &FI, Value *FreedOp)
Instruction * visitCallBrInst(CallBrInst &CBI)
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Instruction * visitFenceInst(FenceInst &FI)
Instruction * visitInvokeInst(InvokeInst &II)
bool SimplifyDemandedInstructionBits(Instruction &Inst)
Tries to simplify operands to an integer instruction based on its demanded bits.
void CreateNonTerminatorUnreachable(Instruction *InsertAt)
Create and insert the idiom we use to indicate a block is unreachable without having to rewrite the C...
Instruction * visitVAEndInst(VAEndInst &I)
Instruction * visitAllocSite(Instruction &FI)
Instruction * SimplifyAnyMemTransfer(AnyMemTransferInst *MI)
OverflowResult computeOverflow(Instruction::BinaryOps BinaryOp, bool IsSigned, Value *LHS, Value *RHS, Instruction *CxtI) const
Instruction * visitCallInst(CallInst &CI)
CallInst simplification.
const DataLayout & getDataLayout() const
DominatorTree & getDominatorTree() const
bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero=false, unsigned Depth=0, const Instruction *CxtI=nullptr)
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
static bool isFreeToInvert(Value *V, bool WillInvertAllUses)
Return true if the specified value is free to invert (apply ~ to).
void replaceUse(Use &U, Value *NewValue)
Replace use and add the previously used value to the worklist.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Instruction * InsertNewInstBefore(Instruction *New, Instruction &Old)
Inserts an instruction New before instruction Old.
std::optional< Instruction * > targetInstCombineIntrinsic(IntrinsicInst &II)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.