27#include "llvm/IR/IntrinsicsAMDGPU.h"
35#define DEBUG_TYPE "AMDGPUtti"
39struct AMDGPUImageDMaskIntrinsic {
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
73 bool AllowI16SExt =
false) {
74 Type *VTy =
V.getType();
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo =
true;
93 APInt IntValue(ConstInt->getValue());
102 Value *CastCandidate;
109 if (!IsExt && !IsFloat && AllowI16SExt)
122 Type *VTy =
V.getType();
131 return Builder.CreateExtractElement(VecCast->
getOperand(0), Idx);
155 Func(Args, OverloadTys);
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
180static std::optional<Instruction *>
188 if (
const auto *LZMappingInfo =
190 if (
auto *ConstantLod =
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
197 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
205 if (
const auto *MIPMappingInfo =
207 if (
auto *ConstantMip =
209 if (ConstantMip->isZero()) {
214 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
222 if (
const auto *BiasMappingInfo =
224 if (
auto *ConstantBias =
226 if (ConstantBias->isZero()) {
231 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
240 if (
const auto *OffsetMappingInfo =
242 if (
auto *ConstantOffset =
244 if (ConstantOffset->isZero()) {
247 OffsetMappingInfo->NoOffset, ImageDimIntr->
Dim);
249 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
266 (DimInfo->
MSAA ? 1 : 0);
268 if (ConstantSlice && ConstantSlice->isZero()) {
273 [&](
auto &Args,
auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
281 if (ST->hasD16Images()) {
287 if (
II.hasOneUse()) {
290 if (
User->getOpcode() == Instruction::FPTrunc &&
294 [&](
auto &Args,
auto &ArgTys) {
297 ArgTys[0] = User->getType();
306 bool AllHalfExtracts =
true;
308 for (
User *U :
II.users()) {
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts =
false;
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts =
false;
324 if (!ExtractTruncPairs.
empty() && AllHalfExtracts) {
335 OverloadTys[0] = HalfVecTy;
338 M, ImageDimIntr->
Intr, OverloadTys);
340 II.mutateType(HalfVecTy);
341 II.setCalledFunction(HalfDecl);
344 for (
auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
347 Builder.SetInsertPoint(Tr);
349 Value *HalfExtract = Builder.CreateExtractElement(&
II, Idx);
352 Tr->replaceAllUsesWith(HalfExtract);
355 for (
auto &[Ext, Tr] : ExtractTruncPairs) {
366 if (!ST->hasA16() && !ST->hasG16())
371 bool HasSampler = BaseOpcode->
Sampler;
372 bool FloatCoord =
false;
374 bool OnlyDerivatives =
false;
379 bool AllowI16SExt = !HasSampler;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord =
II.getOperand(OperandIndex);
386 if (OperandIndex < ImageDimIntr->CoordStart ||
391 OnlyDerivatives =
true;
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives =
true;
404 if (!OnlyDerivatives && ImageDimIntr->
NumBiasArgs != 0) {
407 "Only image instructions with a sampler can have a bias");
409 OnlyDerivatives =
true;
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->
GradientStart ==
420 II,
II,
II.getIntrinsicID(), IC, [&](
auto &Args,
auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(II.getContext());
433 OperandIndex < EndIndex; OperandIndex++) {
435 convertTo16Bit(*II.getOperand(OperandIndex), IC.Builder);
440 Value *Bias = II.getOperand(ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(*Bias, IC.Builder);
475 if (
I.hasNoSignedZeros() &&
485 Value *Src =
nullptr;
488 if (Src->getType()->isHalfTy())
505 unsigned VWidth = VTy->getNumElements();
508 for (
int i = VWidth - 1; i > 0; --i) {
530 unsigned VWidth = VTy->getNumElements();
536 SVI->getShuffleMask(ShuffleMask);
538 for (
int I = VWidth - 1;
I > 0; --
I) {
539 if (ShuffleMask.empty()) {
567 unsigned LaneArgIdx)
const {
568 unsigned MaskBits = ST->getWavefrontSizeLog2();
575 if (!
Known.isConstant())
582 Value *LaneArg =
II.getArgOperand(LaneArgIdx);
584 ConstantInt::get(LaneArg->
getType(),
Known.getConstant() & DemandedMask);
585 if (MaskedConst != LaneArg) {
586 II.getOperandUse(LaneArgIdx).set(MaskedConst);
596 unsigned ScaleArgIdx) {
597 Value *Scale =
II.getArgOperand(ScaleArgIdx);
599 if (StrippedSign != Scale) {
605 if (!BC || !BC->hasOneUse() || !BC->getSrcTy()->isIntegerTy(32))
618 CallInst *NewCall =
B.CreateCall(&NewCallee,
Ops, OpBundles);
634 if (ST.isWave32() &&
match(
V, W32Pred))
636 if (ST.isWave64() &&
match(
V, W64Pred))
643 if (!
I || !ST.isWaveSizeKnown())
647 if (!ST.hasWavefrontsEvenlySplittingXDim(
F,
true))
650 unsigned WaveSize = ST.getWavefrontSize();
655 return match(
V, TidX) && ST.getMaxWorkitemID(
F, 0) < WaveSize;
661 const auto IID =
II.getIntrinsicID();
662 assert(IID == Intrinsic::amdgcn_readlane ||
663 IID == Intrinsic::amdgcn_readfirstlane ||
664 IID == Intrinsic::amdgcn_permlane64);
674 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
678 Value *LaneID =
nullptr;
680 LaneID =
II.getOperand(1);
694 const auto DoIt = [&](
unsigned OpIdx,
698 Ops.push_back(LaneID);
714 return DoIt(0,
II.getCalledFunction());
718 Type *SrcTy = Src->getType();
724 return DoIt(0, Remangled);
732 return DoIt(1,
II.getCalledFunction());
734 return DoIt(0,
II.getCalledFunction());
745 unsigned Depth = 0) {
755 return CI->getZExtValue();
764 std::optional<uint64_t>
LHS =
768 std::optional<uint64_t>
RHS =
777 return CI ? std::optional<uint64_t>(CI->getZExtValue()) : std::nullopt;
785 unsigned WaveSize = ST.getWavefrontSize();
787 for (
unsigned Lane :
seq(WaveSize)) {
789 if (!Val || *Val >= WaveSize)
791 Ids[Lane] =
static_cast<uint8_t>(*Val);
798template <
unsigned Period>
800 static_assert(
isPowerOf2_32(Period),
"Period must be a power of two");
801 for (
size_t I = Period,
E = Ids.
size();
I <
E; ++
I)
802 if (Ids[
I] != Ids[
I % Period] + (
I & ~(Period - 1)))
810 for (
unsigned I = 0;
I <
N; ++
I)
826 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
833 for (
unsigned J = 0; J <
N; ++J)
834 if (Ids[J] != (
N - 1) - J)
846 for (
unsigned J = 1; J < 16; ++J)
847 if (Ids[J] != (Ids[0] + J) % 16)
865 unsigned Mask = Ids[0];
868 for (
unsigned J = 0; J < 16; ++J)
869 if (Ids[J] != (Mask ^ J))
879 unsigned Selector = 0;
880 for (
unsigned J = 0; J < 8; ++J)
881 Selector |= Ids[J] << (J * 3);
890 for (
unsigned J = 0; J < 16; ++J)
891 Sel |=
static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
898 if (Ids.
size() != 64)
900 for (
unsigned J = 0; J < 64; ++J)
901 if (Ids[J] != (J ^ 32))
912 for (
unsigned J = 0; J < 16; ++J) {
913 if (Ids[J] < 16 || Ids[J] >= 32)
915 if (Ids[J + 16] != Ids[J] - 16)
926static std::optional<unsigned>
935 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
936 for (
unsigned B = 0;
B < 5; ++
B) {
937 unsigned Bit0 = (Ids[0] >>
B) & 1;
938 unsigned Bit1 = (Ids[1ull <<
B] >>
B) & 1;
941 XorMask |= Bit0 <<
B;
949 for (
unsigned I :
seq(32u)) {
950 unsigned Expected = ((
I & AndMask) | OrMask) ^ XorMask;
965static std::optional<unsigned>
976 for (
unsigned I = 0;
I < 32; ++
I)
977 if (Ids[
I] != (
I +
N) % 32)
989 return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
991 B.getInt32(0xF),
B.getInt32(0xF),
B.getTrue()});
996 return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->
getType()},
997 {Val,
B.getInt32(Selector)});
1004 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane16, {Ty},
1006 B.getInt32(
Hi),
B.getFalse(),
B.getFalse()});
1014 return B.CreateIntrinsic(Intrinsic::amdgcn_permlanex16, {Ty},
1016 B.getInt32(
Hi),
B.getFalse(),
B.getFalse()});
1024 assert(
DL.getTypeSizeInBits(OrigTy) == 32 &&
1025 "ds_swizzle only supports 32-bit operands");
1029 Src =
B.CreatePtrToInt(Src, I32Ty);
1030 else if (OrigTy != I32Ty)
1031 Src =
B.CreateBitCast(Src, I32Ty);
1032 Value *Result =
B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
1035 return B.CreateIntToPtr(Result, OrigTy);
1036 if (OrigTy != I32Ty)
1037 return B.CreateBitCast(Result, OrigTy);
1043 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->
getType()},
1054 [](
const auto &
E) {
return E.value() ==
E.index(); }))
1078 if (ST.hasDPPRowShare()) {
1083 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1093 if (ST.hasPermlane16Insts()) {
1113 if (ST.hasDsSwizzleRotateMode()) {
1126static std::optional<Instruction *>
1130 if (
DL.getTypeSizeInBits(
II.getType()) != 32)
1131 return std::nullopt;
1133 if (!ST.isWaveSizeKnown())
1134 return std::nullopt;
1136 unsigned WaveSize = ST.getWavefrontSize();
1137 bool IsBpermute =
II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1138 Value *Src =
II.getArgOperand(IsBpermute ? 1 : 0);
1139 Value *Index =
II.getArgOperand(IsBpermute ? 0 : 1);
1144 for (
unsigned Lane :
seq(WaveSize)) {
1146 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1147 return std::nullopt;
1148 Ids[Lane] =
static_cast<uint8_t>(*Val >> 2);
1152 return std::nullopt;
1157 return std::nullopt;
1171 const APInt *Acc =
nullptr;
1179 Value *AccumDelta =
nullptr;
1183 Value *NewAcc =
nullptr;
1184 const APInt *AccumDeltaC =
nullptr;
1186 NewAcc = AccumDelta;
1188 NewAcc = ConstantInt::get(
II.getType(), *Acc + *AccumDeltaC);
1202std::optional<Instruction *>
1206 case Intrinsic::amdgcn_implicitarg_ptr: {
1207 if (
II.getFunction()->hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
1209 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(*
II.getFunction());
1211 uint64_t CurrentOrNullBytes =
1212 II.getAttributes().getRetDereferenceableOrNullBytes();
1213 if (CurrentOrNullBytes != 0) {
1216 uint64_t NewBytes = std::max(CurrentOrNullBytes, ImplicitArgBytes);
1219 II.removeRetAttr(Attribute::DereferenceableOrNull);
1223 uint64_t CurrentBytes =
II.getAttributes().getRetDereferenceableBytes();
1224 uint64_t NewBytes = std::max(CurrentBytes, ImplicitArgBytes);
1225 if (NewBytes != CurrentBytes) {
1231 return std::nullopt;
1233 case Intrinsic::amdgcn_rcp: {
1234 Value *Src =
II.getArgOperand(0);
1245 if (
II.isStrictFP())
1263 auto IID = SrcCI->getIntrinsicID();
1268 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1278 SrcCI->getModule(), Intrinsic::amdgcn_rsq, {SrcCI->getType()});
1281 II.setFastMathFlags(InnerFMF);
1283 II.setCalledFunction(NewDecl);
1289 case Intrinsic::amdgcn_sqrt:
1290 case Intrinsic::amdgcn_rsq:
1291 case Intrinsic::amdgcn_tanh: {
1292 Value *Src =
II.getArgOperand(0);
1304 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1306 II.getModule(), Intrinsic::sqrt, {II.getType()});
1307 II.setCalledFunction(NewDecl);
1313 case Intrinsic::amdgcn_log:
1314 case Intrinsic::amdgcn_exp2: {
1315 const bool IsLog = IID == Intrinsic::amdgcn_log;
1316 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1317 Value *Src =
II.getArgOperand(0);
1327 if (
C->isInfinity()) {
1330 if (!
C->isNegative())
1334 if (IsExp &&
C->isNegative())
1338 if (
II.isStrictFP())
1342 Constant *Quieted = ConstantFP::get(Ty,
C->getValue().makeQuiet());
1347 if (
C->isZero() || (
C->getValue().isDenormal() && Ty->isFloatTy())) {
1349 : ConstantFP::get(Ty, 1.0);
1353 if (IsLog &&
C->isNegative())
1361 case Intrinsic::amdgcn_frexp_mant:
1362 case Intrinsic::amdgcn_frexp_exp: {
1363 Value *Src =
II.getArgOperand(0);
1369 if (IID == Intrinsic::amdgcn_frexp_mant) {
1371 II, ConstantFP::get(
II.getContext(), Significand));
1391 case Intrinsic::amdgcn_class: {
1392 Value *Src0 =
II.getArgOperand(0);
1393 Value *Src1 =
II.getArgOperand(1);
1397 II.getModule(), Intrinsic::is_fpclass, Src0->getType()));
1400 II.setArgOperand(1, ConstantInt::get(Src1->getType(),
1421 case Intrinsic::amdgcn_cvt_pkrtz: {
1422 auto foldFPTruncToF16RTZ = [](
Value *Arg) ->
Value * {
1435 return ConstantFP::get(HalfTy, Val);
1438 Value *Src =
nullptr;
1440 if (Src->getType()->isHalfTy())
1447 if (
Value *Src0 = foldFPTruncToF16RTZ(
II.getArgOperand(0))) {
1448 if (
Value *Src1 = foldFPTruncToF16RTZ(
II.getArgOperand(1))) {
1458 case Intrinsic::amdgcn_cvt_pknorm_i16:
1459 case Intrinsic::amdgcn_cvt_pknorm_u16:
1460 case Intrinsic::amdgcn_cvt_pk_i16:
1461 case Intrinsic::amdgcn_cvt_pk_u16: {
1462 Value *Src0 =
II.getArgOperand(0);
1463 Value *Src1 =
II.getArgOperand(1);
1475 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1476 Value* Arg =
II.getArgOperand(0);
1490 constexpr size_t ResValsSize = 16;
1491 static constexpr float ResVals[ResValsSize] = {
1492 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1493 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1495 ConstantFP::get(Ty, ResVals[CArg->
getZExtValue() & (ResValsSize - 1)]);
1498 case Intrinsic::amdgcn_ubfe:
1499 case Intrinsic::amdgcn_sbfe: {
1501 Value *Src =
II.getArgOperand(0);
1508 unsigned IntSize = Ty->getIntegerBitWidth();
1513 if ((Width & (IntSize - 1)) == 0) {
1518 if (Width >= IntSize) {
1520 II, 2, ConstantInt::get(CWidth->
getType(), Width & (IntSize - 1)));
1531 ConstantInt::get(COffset->
getType(),
Offset & (IntSize - 1)));
1535 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1537 if (!CWidth || !COffset)
1547 if (
Offset + Width < IntSize) {
1551 RightShift->takeName(&
II);
1558 RightShift->takeName(&
II);
1561 case Intrinsic::amdgcn_exp:
1562 case Intrinsic::amdgcn_exp_row:
1563 case Intrinsic::amdgcn_exp_compr: {
1569 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1571 for (
int I = 0;
I < (IsCompr ? 2 : 4); ++
I) {
1572 if ((!IsCompr && (EnBits & (1 <<
I)) == 0) ||
1573 (IsCompr && ((EnBits & (0x3 << (2 *
I))) == 0))) {
1574 Value *Src =
II.getArgOperand(
I + 2);
1588 case Intrinsic::amdgcn_fmed3: {
1589 Value *Src0 =
II.getArgOperand(0);
1590 Value *Src1 =
II.getArgOperand(1);
1591 Value *Src2 =
II.getArgOperand(2);
1593 for (
Value *Src : {Src0, Src1, Src2}) {
1598 if (
II.isStrictFP())
1635 const APFloat *ConstSrc0 =
nullptr;
1636 const APFloat *ConstSrc1 =
nullptr;
1637 const APFloat *ConstSrc2 =
nullptr;
1642 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->
isPosInfinity();
1662 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->
isPosInfinity();
1685 auto *Quieted = ConstantFP::get(
II.getType(), ConstSrc2->
makeQuiet());
1705 CI->copyFastMathFlags(&
II);
1731 II.setArgOperand(0, Src0);
1732 II.setArgOperand(1, Src1);
1733 II.setArgOperand(2, Src2);
1743 ConstantFP::get(
II.getType(), Result));
1748 if (!ST->hasMed3_16())
1757 IID, {
X->getType()}, {
X,
Y, Z}, &
II,
II.getName());
1765 case Intrinsic::amdgcn_mbcnt_hi:
1770 case Intrinsic::amdgcn_mbcnt_lo: {
1780 IID == Intrinsic::amdgcn_mbcnt_lo && ST->isWave64() ? 32 : 31;
1786 if (std::optional<ConstantRange> ExistingRange =
II.getRange()) {
1787 ComputedRange = ComputedRange.
intersectWith(*ExistingRange);
1788 if (ComputedRange == *ExistingRange)
1792 II.addRangeRetAttr(ComputedRange);
1795 case Intrinsic::amdgcn_ballot: {
1796 Value *Arg =
II.getArgOperand(0);
1801 if (Src->isZero()) {
1806 if (ST->isWave32() &&
II.getType()->getIntegerBitWidth() == 64) {
1813 {IC.Builder.getInt32Ty()},
1814 {II.getArgOperand(0)}),
1821 case Intrinsic::amdgcn_wavefrontsize: {
1822 if (ST->isWaveSizeKnown())
1824 II, ConstantInt::get(
II.getType(), ST->getWavefrontSize()));
1827 case Intrinsic::amdgcn_wqm_vote: {
1834 case Intrinsic::amdgcn_kill: {
1836 if (!
C || !
C->getZExtValue())
1842 case Intrinsic::amdgcn_s_sendmsg:
1843 case Intrinsic::amdgcn_s_sendmsghalt: {
1849 Value *M0Val =
II.getArgOperand(1);
1854 uint16_t MsgId, OpId, StreamId;
1855 decodeMsg(
static_cast<unsigned>(MsgImm->getZExtValue()), MsgId, OpId,
1858 if (!msgDoesNotUseM0(MsgId, *ST))
1862 II.dropUBImplyingAttrsAndMetadata();
1866 case Intrinsic::amdgcn_update_dpp: {
1867 Value *Old =
II.getArgOperand(0);
1872 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1879 case Intrinsic::amdgcn_permlane16:
1880 case Intrinsic::amdgcn_permlane16_var:
1881 case Intrinsic::amdgcn_permlanex16:
1882 case Intrinsic::amdgcn_permlanex16_var: {
1884 Value *VDstIn =
II.getArgOperand(0);
1889 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1890 IID == Intrinsic::amdgcn_permlanex16)
1897 unsigned int BcIdx = FiIdx + 1;
1906 case Intrinsic::amdgcn_wave_shuffle:
1908 case Intrinsic::amdgcn_permlane64:
1909 case Intrinsic::amdgcn_readfirstlane:
1910 case Intrinsic::amdgcn_readlane:
1911 case Intrinsic::amdgcn_ds_bpermute: {
1913 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1914 const Use &Src =
II.getArgOperandUse(SrcIdx);
1918 if (IID == Intrinsic::amdgcn_readlane &&
1925 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1926 const Use &Lane =
II.getArgOperandUse(0);
1930 II.getModule(), Intrinsic::amdgcn_readlane,
II.getType());
1931 II.setCalledFunction(NewDecl);
1932 II.setOperand(0, Src);
1933 II.setOperand(1, NewLane);
1938 if (IID == Intrinsic::amdgcn_ds_bpermute)
1944 return std::nullopt;
1946 case Intrinsic::amdgcn_wave_match_b32: {
1947 const Use &Src0 =
II.getArgOperandUse(0);
1948 const Use &Src1 =
II.getArgOperandUse(1);
1951 II.getModule(), Intrinsic::amdgcn_ballot,
II.getType());
1958 case Intrinsic::amdgcn_writelane: {
1962 return std::nullopt;
1964 case Intrinsic::amdgcn_trig_preop: {
1967 if (!
II.getType()->isDoubleTy())
1970 Value *Src =
II.getArgOperand(0);
1971 Value *Segment =
II.getArgOperand(1);
1980 if (StrippedSign != Src)
1983 if (
II.isStrictFP())
2005 unsigned SegmentVal =
2006 static_cast<unsigned>(Cseg->
getValue().trunc(5).getZExtValue());
2007 unsigned Shift = SegmentVal * 53;
2012 static const uint32_t TwoByPi[] = {
2013 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
2014 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
2015 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
2016 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
2017 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
2018 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
2022 unsigned Idx = Shift >> 5;
2023 if (Idx + 2 >= std::size(TwoByPi)) {
2028 unsigned BShift = Shift & 0x1f;
2029 uint64_t Thi =
Make_64(TwoByPi[Idx], TwoByPi[Idx + 1]);
2030 uint64_t Tlo =
Make_64(TwoByPi[Idx + 2], 0);
2032 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2036 int Scale = -53 - Shift;
2043 case Intrinsic::amdgcn_sdot2:
2044 case Intrinsic::amdgcn_udot2:
2045 case Intrinsic::amdgcn_sdot4:
2046 case Intrinsic::amdgcn_udot4:
2047 case Intrinsic::amdgcn_sdot8:
2048 case Intrinsic::amdgcn_udot8: {
2049 Value *Src0 =
II.getArgOperand(0);
2050 Value *Src1 =
II.getArgOperand(1);
2054 II.setArgOperand(0, Src1);
2055 II.setArgOperand(1, Src0);
2067 case Intrinsic::amdgcn_sudot4:
2068 case Intrinsic::amdgcn_sudot8: {
2069 Value *Src0 =
II.getArgOperand(1);
2070 Value *Src1 =
II.getArgOperand(3);
2075 Value *Sign0 =
II.getArgOperand(0);
2076 Value *Sign1 =
II.getArgOperand(2);
2077 II.setArgOperand(0, Sign1);
2078 II.setArgOperand(1, Src1);
2079 II.setArgOperand(2, Sign0);
2080 II.setArgOperand(3, Src0);
2092 case Intrinsic::amdgcn_fmul_legacy: {
2093 Value *Op0 =
II.getArgOperand(0);
2094 Value *Op1 =
II.getArgOperand(1);
2096 for (
Value *Src : {Op0, Op1}) {
2117 case Intrinsic::amdgcn_fma_legacy: {
2118 Value *Op0 =
II.getArgOperand(0);
2119 Value *Op1 =
II.getArgOperand(1);
2120 Value *Op2 =
II.getArgOperand(2);
2122 for (
Value *Src : {Op0, Op1, Op2}) {
2144 II.getModule(), Intrinsic::fma,
II.getType()));
2149 case Intrinsic::amdgcn_is_shared:
2150 case Intrinsic::amdgcn_is_private: {
2151 Value *Src =
II.getArgOperand(0);
2161 case Intrinsic::amdgcn_make_buffer_rsrc: {
2162 Value *Src =
II.getArgOperand(0);
2167 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2169 return std::nullopt;
2171 if (
II.getArgOperand(2)->getType() == NumRecordsTy)
2172 return std::nullopt;
2176 Intrinsic::amdgcn_make_buffer_rsrc,
2177 {
II.getType(), Src->getType(), NumRecordsTy}, Args);
2183 II.getAttributes().removeParamAttributes(
II.getContext(), 2));
2187 case Intrinsic::amdgcn_raw_buffer_store_format:
2188 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
2189 case Intrinsic::amdgcn_struct_buffer_store_format:
2190 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
2191 case Intrinsic::amdgcn_raw_tbuffer_store:
2192 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
2193 case Intrinsic::amdgcn_struct_tbuffer_store:
2194 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
2195 case Intrinsic::amdgcn_image_store_1d:
2196 case Intrinsic::amdgcn_image_store_1darray:
2197 case Intrinsic::amdgcn_image_store_2d:
2198 case Intrinsic::amdgcn_image_store_2darray:
2199 case Intrinsic::amdgcn_image_store_2darraymsaa:
2200 case Intrinsic::amdgcn_image_store_2dmsaa:
2201 case Intrinsic::amdgcn_image_store_3d:
2202 case Intrinsic::amdgcn_image_store_cube:
2203 case Intrinsic::amdgcn_image_store_mip_1d:
2204 case Intrinsic::amdgcn_image_store_mip_1darray:
2205 case Intrinsic::amdgcn_image_store_mip_2d:
2206 case Intrinsic::amdgcn_image_store_mip_2darray:
2207 case Intrinsic::amdgcn_image_store_mip_3d:
2208 case Intrinsic::amdgcn_image_store_mip_cube: {
2213 if (ST->hasDefaultComponentBroadcast())
2215 else if (ST->hasDefaultComponentZero())
2225 case Intrinsic::amdgcn_prng_b32: {
2226 auto *Src =
II.getArgOperand(0);
2230 return std::nullopt;
2232 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2233 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2234 Value *Src0 =
II.getArgOperand(0);
2235 Value *Src1 =
II.getArgOperand(1);
2241 auto getFormatNumRegs = [](uint64_t FormatVal) {
2242 switch (FormatVal) {
2256 bool MadeChange =
false;
2257 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2258 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2262 if (Src0Ty->getNumElements() > Src0NumElts) {
2269 if (Src1Ty->getNumElements() > Src1NumElts) {
2277 return std::nullopt;
2284 IID, {Src0->getType(), Src1->getType()}, Args, &
II);
2288 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2289 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2290 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2291 Value *Src0 =
II.getArgOperand(1);
2292 Value *Src1 =
II.getArgOperand(3);
2298 bool MadeChange =
false;
2304 if (Src0Ty->getNumElements() > Src0NumElts) {
2311 if (Src1Ty->getNumElements() > Src1NumElts) {
2319 return std::nullopt;
2326 IID, {
II.getArgOperand(5)->getType(), Src0->getType(), Src1->getType()},
2336 if (std::optional<unsigned> ScaleArgIdx =
2341 return std::nullopt;
2355 const bool IsLoad = !
II.getType()->isVoidTy();
2357 if (getAMDGPUImageDMaskIntrinsic(
II.getIntrinsicID()))
2358 DMaskIdx = IsLoad ? 0 : 1;
2361 :
II.getOperand(0)->getType());
2362 unsigned VWidth = IIVTy->getNumElements();
2365 Type *EltTy = IIVTy->getElementType();
2372 uint64_t OffsetAdd = 0;
2378 const unsigned UnusedComponentsAtFront = DemandedElts.
countr_zero();
2383 DemandedElts = (1 << ActiveBits) - 1;
2385 if (UnusedComponentsAtFront > 0) {
2390 bool CanAdjustOffset = BufInfo &&
2391 (BufInfo->
Kind == AMDGPU::BufferKindLoad ||
2392 BufInfo->
Kind == AMDGPU::BufferKindSLoad) &&
2397 if (CanAdjustOffset && BufInfo->
Kind == AMDGPU::BufferKindSLoad &&
2398 ActiveBits == 4 && UnusedComponentsAtFront == 1)
2399 CanAdjustOffset =
false;
2402 unsigned FormatBitsPerComp = 0;
2403 if (CanAdjustOffset && BufInfo->
isTyped()) {
2413 if (OldFormat && ActiveBits <= OldFormat->NumComponents)
2420 ConstantInt::get(Args[FormatIdx]->
getType(), NewFormat->
Format);
2423 CanAdjustOffset =
false;
2427 if (CanAdjustOffset) {
2430 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2431 auto *
Offset = Args[OffsetIdx];
2432 uint64_t SingleComponentSizeInBits =
2436 OffsetAdd = UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2437 auto *OffsetAddVal = ConstantInt::get(
Offset->getType(), OffsetAdd);
2457 unsigned NewDMaskVal = 0;
2458 unsigned OrigLdStIdx = 0;
2459 for (
unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2460 const unsigned Bit = 1 << SrcIdx;
2461 if (!!(DMaskVal & Bit)) {
2462 if (!!DemandedElts[OrigLdStIdx])
2468 if (DMaskVal != NewDMaskVal)
2469 Args[DMaskIdx] = ConstantInt::get(DMask->
getType(), NewDMaskVal);
2472 unsigned NewNumElts = DemandedElts.
popcount();
2476 if (NewNumElts >= VWidth && DemandedElts.
isMask()) {
2478 II.setArgOperand(DMaskIdx, Args[DMaskIdx]);
2490 OverloadTys[0] = NewTy;
2494 for (
unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2495 if (DemandedElts[OrigStoreIdx])
2498 if (NewNumElts == 1)
2505 II.getIntrinsicID(), OverloadTys, Args);
2511 if (OffsetAdd != 0) {
2524 if (NewNumElts == 1) {
2530 unsigned NewLoadIdx = 0;
2531 for (
unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2532 if (!!DemandedElts[OrigLoadIdx])
2548 APInt &UndefElts)
const {
2553 const unsigned FirstElt = DemandedElts.
countr_zero();
2555 const unsigned MaskLen = LastElt - FirstElt + 1;
2557 unsigned OldNumElts = VT->getNumElements();
2558 if (MaskLen == OldNumElts && MaskLen != 1)
2561 Type *EltTy = VT->getElementType();
2569 Value *Src =
II.getArgOperand(0);
2574 II.getOperandBundlesAsDefs(OpBundles);
2591 for (
unsigned I = 0;
I != MaskLen; ++
I) {
2592 if (DemandedElts[FirstElt +
I])
2593 ExtractMask[
I] = FirstElt +
I;
2602 for (
unsigned I = 0;
I != MaskLen; ++
I) {
2603 if (DemandedElts[FirstElt +
I])
2604 InsertMask[FirstElt +
I] =
I;
2616 SimplifyAndSetOp)
const {
2617 switch (
II.getIntrinsicID()) {
2618 case Intrinsic::amdgcn_readfirstlane:
2619 SimplifyAndSetOp(&
II, 0, DemandedElts, UndefElts);
2624 BufInfo && (BufInfo->Kind == AMDGPU::BufferKindLoad ||
2625 BufInfo->Kind == AMDGPU::BufferKindSLoad))
2627 if (getAMDGPUImageDMaskIntrinsic(
II.getIntrinsicID()))
2631 return std::nullopt;
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static Value * createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlane16 with the precomputed lane-select halves.
static std::optional< unsigned > matchRowSharePattern(ArrayRef< uint8_t > Ids)
Match a row-share pattern: all 16 lanes of each row read the same source lane.
static bool matchMirrorPattern(ArrayRef< uint8_t > Ids)
Match an N-lane reversal (mirror) pattern.
static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat, bool AllowI16SExt=false)
static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST, SmallVectorImpl< uint8_t > &Ids, const DataLayout &DL)
Build the per-lane shuffle map by evaluating Index for every lane in the wave.
static std::optional< unsigned > matchQuadPermPattern(ArrayRef< uint8_t > Ids)
Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp QUAD_PERM control word: bits[1:0]=Ids...
static std::optional< unsigned > matchDsSwizzleRotatePattern(ArrayRef< uint8_t > Ids)
Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation of all 32 lanes within each ...
static std::optional< unsigned > matchHalfRowPermPattern(ArrayRef< uint8_t > Ids)
Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8 24-bit selector (three bits per ...
static std::optional< unsigned > matchRowXMaskPattern(ArrayRef< uint8_t > Ids)
Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J, with Mask in [1,...
static constexpr auto matchHalfRowMirrorPattern
static Value * createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlanex16 with the precomputed lane-select halves.
static bool isRowPattern(ArrayRef< uint8_t > Ids)
Match an N-lane row pattern: each lane in [0, N) reads from a source lane in the same N-lane row,...
static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp)
Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
static CallInst * rewriteCall(IRBuilderBase &B, CallInst &Old, Function &NewCallee, ArrayRef< Value * > Ops)
static Value * convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder)
static constexpr auto isFullRowPattern
static constexpr auto isQuadPattern
static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV, Instruction *I)
static uint64_t computePermlane16Masks(ArrayRef< uint8_t > Ids)
Pack a 16-lane permutation into a single 64-bit value: four bits per output lane, lane J in bits [J*4...
static bool matchHalfWaveSwapPattern(ArrayRef< uint8_t > Ids)
Match a half-wave swap: lane J reads from lane J ^ 32.
static bool hasPeriodicLayout(ArrayRef< uint8_t > Ids)
Lanes are partitioned into groups of Period; each group is a translated copy of the first: Ids[I] = I...
static std::optional< uint64_t > evalLaneExpr(Value *V, unsigned Lane, const GCNSubtarget &ST, const DataLayout &DL, unsigned Depth=0)
Evaluate V as a function of the lane ID and return its value on Lane, or std::nullopt if V is not a c...
static std::optional< Instruction * > tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II, const GCNSubtarget &ST)
Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant function of the lane ID into a ...
static constexpr auto isHalfRowPattern
static APInt defaultComponentBroadcast(Value *V)
static std::optional< unsigned > matchDsSwizzleBitmaskPattern(ArrayRef< uint8_t > Ids)
Match a DS_SWIZZLE bitmask-mode permutation: dst_lane = ((src_lane & AND) | OR) ^ XOR with each mask ...
static Value * createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset, const DataLayout &DL)
Emit ds_swizzle with the given immediate, bitcasting/converting between pointer/float types and i32 a...
static std::optional< Instruction * > modifyIntrinsicCall(IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr, InstCombiner &IC, std::function< void(SmallVectorImpl< Value * > &, SmallVectorImpl< Type * > &)> Func)
Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with modified arguments (based on ...
static Value * matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src, ArrayRef< uint8_t > Ids, const GCNSubtarget &ST, const DataLayout &DL)
Given a shuffle map, try to emit the best hardware intrinsic.
static std::optional< unsigned > matchRowRotatePattern(ArrayRef< uint8_t > Ids)
Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
static bool isCrossRowPattern(ArrayRef< uint8_t > Ids)
Match a cross-row permutation suitable for v_permlanex16: every lane in the low 16-lane half reads fr...
static bool isThreadID(const GCNSubtarget &ST, Value *V)
static Value * createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl)
Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and bound_ctrl=1 so out-of-bounds...
static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1, const APFloat &Src2)
static std::optional< Instruction * > simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr, IntrinsicInst &II, InstCombiner &IC)
static Value * createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector)
Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
static Instruction * foldAddIntoDotAccumulator(IntrinsicInst &II, unsigned AccIdx, unsigned ClampIdx, InstCombiner &IC)
Try to fold an addition into the accumulator when saturation is disabled.
static Value * matchFPExtFromF16(Value *Arg)
Match an fpext from half to float, or a constant we can convert.
static constexpr auto matchFullRowMirrorPattern
static bool simplifyExponentOnlyScaleArg(InstCombiner &IC, IntrinsicInst &II, unsigned ScaleArgIdx)
Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which only the exponent field is ...
static Value * createPermlane64(IRBuilderBase &B, Value *Val)
Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
uint64_t IntrinsicInst * II
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
static const fltSemantics & IEEEsingle()
static constexpr roundingMode rmTowardZero
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
bool bitwiseIsEqual(const APFloat &RHS) const
bool isPosInfinity() const
APFloat makeQuiet() const
Assuming this is an IEEE-754 NaN value, quiet its signaling bit.
APInt bitcastToAPInt() const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
unsigned popcount() const
Count the number of bits set.
LLVM_ABI uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isMask(unsigned numBits) const
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
size_t size() const
Get the array size.
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
bool isTypeLegal(Type *Ty) const override
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
void setAttributes(AttributeList A)
Set the attributes for this call.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
static LLVM_ABI ConstantFP * getNaN(Type *Ty, bool Negative=false, uint64_t Payload=0)
static LLVM_ABI ConstantFP * getInfinity(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
This is an important base class in LLVM.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Tagged union holding either a T or a Error.
This class represents an extension of floating point types.
Utility class for floating point operations which can have information about relaxed accuracy require...
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
bool hasApproxFunc() const
Test if this operation allows approximations of math library functions or intrinsics.
LLVM_ABI float getFPAccuracy() const
Get the maximum error permitted by this operation in ULPs.
Convenience struct for specifying and reasoning about fast-math flags.
bool allowContract() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II, unsigned LaneAgIdx) const
Simplify a lane index operand (e.g.
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override
Instruction * hoistLaneIntrinsicThroughOperand(InstCombiner &IC, IntrinsicInst &II) const
Value * simplifyAMDGCNMemoryIntrinsicDemanded(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts) const
Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const override
KnownIEEEMode fpenvIEEEMode(const Instruction &I) const
Return KnownIEEEMode::On if we know if the use context can assume "amdgpu-ieee"="true" and KnownIEEEM...
Value * simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts, APInt &UndefElts) const
bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0, const Value *Op1, InstCombiner &IC) const
Common base class shared among various IRBuilders.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
LLVM_ABI CallInst * CreateIntrinsicWithoutFolding(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={})
Create a call to intrinsic ID with Args, mangled using OverloadTypes.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name="")
Create a call to the vector.extract intrinsic.
Module * getModule() const
Get the module.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateMaxNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the maxnum intrinsic.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
Value * CreateMaximumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the maximum intrinsic.
Value * CreateMinNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the minnum intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateFAddFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateMinimumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the minimumnum intrinsic.
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateFMulFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
The core instruction combiner logic.
const DataLayout & getDataLayout() const
virtual Instruction * eraseInstFromFunction(Instruction &I)=0
Combiner aware instruction erasure.
DominatorTree & getDominatorTree() const
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
virtual bool SimplifyDemandedBits(Instruction *I, unsigned OpNo, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0)=0
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
static Value * stripSignOnlyFPOps(Value *Val)
Ignore all operations which only change the sign of a value, returning the underlying magnitude value...
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
void dropPoisonGeneratingAnnotations()
Drops flags, attributes and metadata that may generate poison.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverInfOrNaN() const
Return true if it's known this can never be an infinity or nan.
LLVM_ABI bool isKnownNeverLogicalZero(DenormalMode Mode) const
Return true if it's known this can never be interpreted as a zero.
A Module instance is used to store all the information related to an LLVM module.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt getExponentOnlyScaleDemandedBits()
LLVM_READONLY const MIMGOffsetMappingInfo * getMIMGOffsetMappingInfo(unsigned Offset)
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const ImageDimIntrinsicInfo * getImageDimIntrinsicByBaseOpcode(unsigned BaseOpcode, unsigned Dim)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
LLVM_READONLY const MIMGMIPMappingInfo * getMIMGMIPMappingInfo(unsigned MIP)
bool isTriviallyUniform(const Value *V, const Instruction *CtxI)
Return true if we can easily prove that V is uniform at CtxI.
const BufferIntrinsicInfo * lookupBufferIntrinsic(unsigned Intr)
LLVM_READONLY const MIMGBiasMappingInfo * getMIMGBiasMappingInfo(unsigned Bias)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
LLVM_READONLY const MIMGLZMappingInfo * getMIMGLZMappingInfo(unsigned L)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_ConstantFP()
Match an arbitrary ConstantFP and ignore it.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
@ Known
Known to have no common set bits.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
auto dyn_cast_or_null(const Y &Val)
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
constexpr unsigned MaxAnalysisRecursionDepth
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
constexpr int PoisonMaskElem
LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
@ NearestTiesToEven
roundTiesToEven.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
constexpr uint64_t Make_64(uint32_t High, uint32_t Low)
Make a 64-bit integer from a high / low pair of 32-bit integers.
LLVM_ABI ConstantRange computeConstantRange(const Value *V, bool ForSigned, const SimplifyQuery &SQ, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
The argument layout of a buffer intrinsic.
bool isFormatFromRsrc() const
Whether the data is converted according to the format in the resource.
unsigned getOffsetArg() const
The main offset: the voffset if there is one, and the soffset otherwise.
std::optional< unsigned > getFormatArg() const
Represent subnormal handling kind for floating point instruction inputs and outputs.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
SimplifyQuery getWithInstruction(const Instruction *I) const
LLVM_ABI bool isUndefValue(Value *V) const
If CanUseUndef is true, returns whether V is undef.