27#include "llvm/IR/IntrinsicsAMDGPU.h"
35#define DEBUG_TYPE "AMDGPUtti"
39struct AMDGPUImageDMaskIntrinsic {
43#define GET_AMDGPUImageDMaskIntrinsicTable_IMPL
44#include "AMDGPUGenSearchableTables.inc"
54 assert(!Src0.isNaN() && !Src1.isNaN() && !Src2.isNaN() &&
55 "nans handled separately");
73 bool AllowI16SExt =
false) {
74 Type *VTy =
V.getType();
83 APFloat FloatValue(ConstFloat->getValueAPF());
84 bool LosesInfo =
true;
93 APInt IntValue(ConstInt->getValue());
102 Value *CastCandidate;
109 if (!IsExt && !IsFloat && AllowI16SExt)
122 Type *VTy =
V.getType();
131 return Builder.CreateExtractElement(VecCast->
getOperand(0), Idx);
155 Func(Args, OverloadTys);
171 bool RemoveOldIntr = &OldIntr != &InstToReplace;
180static std::optional<Instruction *>
188 if (
const auto *LZMappingInfo =
190 if (
auto *ConstantLod =
192 if (ConstantLod->isZero() || ConstantLod->isNegative()) {
197 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
198 Args.erase(Args.begin() + ImageDimIntr->LodIndex);
205 if (
const auto *MIPMappingInfo =
207 if (
auto *ConstantMip =
209 if (ConstantMip->isZero()) {
214 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
215 Args.erase(Args.begin() + ImageDimIntr->MipIndex);
222 if (
const auto *BiasMappingInfo =
224 if (
auto *ConstantBias =
226 if (ConstantBias->isZero()) {
231 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
232 Args.erase(Args.begin() + ImageDimIntr->BiasIndex);
233 ArgTys.erase(ArgTys.begin() + ImageDimIntr->BiasTyArg);
240 if (
const auto *OffsetMappingInfo =
242 if (
auto *ConstantOffset =
244 if (ConstantOffset->isZero()) {
247 OffsetMappingInfo->NoOffset, ImageDimIntr->
Dim);
249 II,
II, NewImageDimIntr->
Intr, IC, [&](
auto &Args,
auto &ArgTys) {
250 Args.erase(Args.begin() + ImageDimIntr->OffsetIndex);
266 (DimInfo->
MSAA ? 1 : 0);
268 if (ConstantSlice && ConstantSlice->isZero()) {
273 [&](
auto &Args,
auto &ArgTys) {
274 Args.erase(Args.begin() + SliceIndex);
281 if (ST->hasD16Images()) {
287 if (
II.hasOneUse()) {
290 if (
User->getOpcode() == Instruction::FPTrunc &&
294 [&](
auto &Args,
auto &ArgTys) {
297 ArgTys[0] = User->getType();
306 bool AllHalfExtracts =
true;
308 for (
User *U :
II.users()) {
310 if (!Ext || !Ext->hasOneUse()) {
311 AllHalfExtracts =
false;
316 if (!Tr || !Tr->getType()->isHalfTy()) {
317 AllHalfExtracts =
false;
324 if (!ExtractTruncPairs.
empty() && AllHalfExtracts) {
335 OverloadTys[0] = HalfVecTy;
338 M, ImageDimIntr->
Intr, OverloadTys);
340 II.mutateType(HalfVecTy);
341 II.setCalledFunction(HalfDecl);
344 for (
auto &[Ext, Tr] : ExtractTruncPairs) {
345 Value *Idx = Ext->getIndexOperand();
347 Builder.SetInsertPoint(Tr);
349 Value *HalfExtract = Builder.CreateExtractElement(&
II, Idx);
352 Tr->replaceAllUsesWith(HalfExtract);
355 for (
auto &[Ext, Tr] : ExtractTruncPairs) {
366 if (!ST->hasA16() && !ST->hasG16())
371 bool HasSampler = BaseOpcode->
Sampler;
372 bool FloatCoord =
false;
374 bool OnlyDerivatives =
false;
379 bool AllowI16SExt = !HasSampler;
382 OperandIndex < ImageDimIntr->VAddrEnd; OperandIndex++) {
383 Value *Coord =
II.getOperand(OperandIndex);
386 if (OperandIndex < ImageDimIntr->CoordStart ||
391 OnlyDerivatives =
true;
400 if (!OnlyDerivatives && !ST->hasA16())
401 OnlyDerivatives =
true;
404 if (!OnlyDerivatives && ImageDimIntr->
NumBiasArgs != 0) {
407 "Only image instructions with a sampler can have a bias");
409 OnlyDerivatives =
true;
412 if (OnlyDerivatives && (!ST->hasG16() || ImageDimIntr->
GradientStart ==
420 II,
II,
II.getIntrinsicID(), IC, [&](
auto &Args,
auto &ArgTys) {
421 ArgTys[ImageDimIntr->GradientTyArg] = CoordType;
422 if (!OnlyDerivatives) {
423 ArgTys[ImageDimIntr->CoordTyArg] = CoordType;
426 if (ImageDimIntr->NumBiasArgs != 0)
427 ArgTys[ImageDimIntr->BiasTyArg] = Type::getHalfTy(II.getContext());
433 OperandIndex < EndIndex; OperandIndex++) {
435 convertTo16Bit(*II.getOperand(OperandIndex), IC.Builder);
440 Value *Bias = II.getOperand(ImageDimIntr->BiasIndex);
441 Args[ImageDimIntr->BiasIndex] = convertTo16Bit(*Bias, IC.Builder);
475 if (
I.hasNoSignedZeros() &&
485 Value *Src =
nullptr;
488 if (Src->getType()->isHalfTy())
505 unsigned VWidth = VTy->getNumElements();
508 for (
int i = VWidth - 1; i > 0; --i) {
530 unsigned VWidth = VTy->getNumElements();
536 SVI->getShuffleMask(ShuffleMask);
538 for (
int I = VWidth - 1;
I > 0; --
I) {
539 if (ShuffleMask.empty()) {
558 int DMaskIdx = -1,
bool IsLoad =
true);
572 unsigned LaneArgIdx)
const {
573 unsigned MaskBits = ST->getWavefrontSizeLog2();
580 if (!
Known.isConstant())
587 Value *LaneArg =
II.getArgOperand(LaneArgIdx);
589 ConstantInt::get(LaneArg->
getType(),
Known.getConstant() & DemandedMask);
590 if (MaskedConst != LaneArg) {
591 II.getOperandUse(LaneArgIdx).set(MaskedConst);
601 unsigned ScaleArgIdx) {
602 Value *Scale =
II.getArgOperand(ScaleArgIdx);
604 if (StrippedSign != Scale) {
610 if (!BC || !BC->hasOneUse() || !BC->getSrcTy()->isIntegerTy(32))
623 CallInst *NewCall =
B.CreateCall(&NewCallee,
Ops, OpBundles);
639 if (ST.isWave32() &&
match(
V, W32Pred))
641 if (ST.isWave64() &&
match(
V, W64Pred))
648 if (!
I || !ST.isWaveSizeKnown())
652 if (!ST.hasWavefrontsEvenlySplittingXDim(
F,
true))
655 unsigned WaveSize = ST.getWavefrontSize();
660 return match(
V, TidX) && ST.getMaxWorkitemID(
F, 0) < WaveSize;
666 const auto IID =
II.getIntrinsicID();
667 assert(IID == Intrinsic::amdgcn_readlane ||
668 IID == Intrinsic::amdgcn_readfirstlane ||
669 IID == Intrinsic::amdgcn_permlane64);
679 const bool IsReadLane = (IID == Intrinsic::amdgcn_readlane);
683 Value *LaneID =
nullptr;
685 LaneID =
II.getOperand(1);
699 const auto DoIt = [&](
unsigned OpIdx,
703 Ops.push_back(LaneID);
719 return DoIt(0,
II.getCalledFunction());
723 Type *SrcTy = Src->getType();
729 return DoIt(0, Remangled);
737 return DoIt(1,
II.getCalledFunction());
739 return DoIt(0,
II.getCalledFunction());
750 unsigned Depth = 0) {
760 return CI->getZExtValue();
769 std::optional<uint64_t>
LHS =
773 std::optional<uint64_t>
RHS =
782 return CI ? std::optional<uint64_t>(CI->getZExtValue()) : std::nullopt;
790 unsigned WaveSize = ST.getWavefrontSize();
792 for (
unsigned Lane :
seq(WaveSize)) {
794 if (!Val || *Val >= WaveSize)
796 Ids[Lane] =
static_cast<uint8_t>(*Val);
803template <
unsigned Period>
805 static_assert(
isPowerOf2_32(Period),
"Period must be a power of two");
806 for (
size_t I = Period,
E = Ids.
size();
I <
E; ++
I)
807 if (Ids[
I] != Ids[
I % Period] + (
I & ~(Period - 1)))
815 for (
unsigned I = 0;
I <
N; ++
I)
831 return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
838 for (
unsigned J = 0; J <
N; ++J)
839 if (Ids[J] != (
N - 1) - J)
851 for (
unsigned J = 1; J < 16; ++J)
852 if (Ids[J] != (Ids[0] + J) % 16)
870 unsigned Mask = Ids[0];
873 for (
unsigned J = 0; J < 16; ++J)
874 if (Ids[J] != (Mask ^ J))
884 unsigned Selector = 0;
885 for (
unsigned J = 0; J < 8; ++J)
886 Selector |= Ids[J] << (J * 3);
895 for (
unsigned J = 0; J < 16; ++J)
896 Sel |=
static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
903 if (Ids.
size() != 64)
905 for (
unsigned J = 0; J < 64; ++J)
906 if (Ids[J] != (J ^ 32))
917 for (
unsigned J = 0; J < 16; ++J) {
918 if (Ids[J] < 16 || Ids[J] >= 32)
920 if (Ids[J + 16] != Ids[J] - 16)
931static std::optional<unsigned>
940 unsigned AndMask = 0, OrMask = 0, XorMask = 0;
941 for (
unsigned B = 0;
B < 5; ++
B) {
942 unsigned Bit0 = (Ids[0] >>
B) & 1;
943 unsigned Bit1 = (Ids[1ull <<
B] >>
B) & 1;
946 XorMask |= Bit0 <<
B;
954 for (
unsigned I :
seq(32u)) {
955 unsigned Expected = ((
I & AndMask) | OrMask) ^ XorMask;
970static std::optional<unsigned>
981 for (
unsigned I = 0;
I < 32; ++
I)
982 if (Ids[
I] != (
I +
N) % 32)
994 return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
996 B.getInt32(0xF),
B.getInt32(0xF),
B.getTrue()});
1001 return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->
getType()},
1002 {Val,
B.getInt32(Selector)});
1009 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane16, {Ty},
1011 B.getInt32(
Hi),
B.getFalse(),
B.getFalse()});
1019 return B.CreateIntrinsic(Intrinsic::amdgcn_permlanex16, {Ty},
1021 B.getInt32(
Hi),
B.getFalse(),
B.getFalse()});
1029 assert(
DL.getTypeSizeInBits(OrigTy) == 32 &&
1030 "ds_swizzle only supports 32-bit operands");
1034 Src =
B.CreatePtrToInt(Src, I32Ty);
1035 else if (OrigTy != I32Ty)
1036 Src =
B.CreateBitCast(Src, I32Ty);
1037 Value *Result =
B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
1040 return B.CreateIntToPtr(Result, OrigTy);
1041 if (OrigTy != I32Ty)
1042 return B.CreateBitCast(Result, OrigTy);
1048 return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->
getType()},
1059 [](
const auto &
E) {
return E.value() ==
E.index(); }))
1083 if (ST.hasDPPRowShare()) {
1088 if (ST.hasDPP() && ST.hasGFX10Insts()) {
1098 if (ST.hasPermlane16Insts()) {
1118 if (ST.hasDsSwizzleRotateMode()) {
1131static std::optional<Instruction *>
1135 if (
DL.getTypeSizeInBits(
II.getType()) != 32)
1136 return std::nullopt;
1138 if (!ST.isWaveSizeKnown())
1139 return std::nullopt;
1141 unsigned WaveSize = ST.getWavefrontSize();
1142 bool IsBpermute =
II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
1143 Value *Src =
II.getArgOperand(IsBpermute ? 1 : 0);
1144 Value *Index =
II.getArgOperand(IsBpermute ? 0 : 1);
1149 for (
unsigned Lane :
seq(WaveSize)) {
1151 if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
1152 return std::nullopt;
1153 Ids[Lane] =
static_cast<uint8_t>(*Val >> 2);
1157 return std::nullopt;
1162 return std::nullopt;
1176 const APInt *Acc =
nullptr;
1184 Value *AccumDelta =
nullptr;
1188 Value *NewAcc =
nullptr;
1189 const APInt *AccumDeltaC =
nullptr;
1191 NewAcc = AccumDelta;
1193 NewAcc = ConstantInt::get(
II.getType(), *Acc + *AccumDeltaC);
1207std::optional<Instruction *>
1211 case Intrinsic::amdgcn_implicitarg_ptr: {
1212 if (
II.getFunction()->hasFnAttribute(
"amdgpu-no-implicitarg-ptr"))
1214 uint64_t ImplicitArgBytes = ST->getImplicitArgNumBytes(*
II.getFunction());
1216 uint64_t CurrentOrNullBytes =
1217 II.getAttributes().getRetDereferenceableOrNullBytes();
1218 if (CurrentOrNullBytes != 0) {
1221 uint64_t NewBytes = std::max(CurrentOrNullBytes, ImplicitArgBytes);
1224 II.removeRetAttr(Attribute::DereferenceableOrNull);
1228 uint64_t CurrentBytes =
II.getAttributes().getRetDereferenceableBytes();
1229 uint64_t NewBytes = std::max(CurrentBytes, ImplicitArgBytes);
1230 if (NewBytes != CurrentBytes) {
1236 return std::nullopt;
1238 case Intrinsic::amdgcn_rcp: {
1239 Value *Src =
II.getArgOperand(0);
1250 if (
II.isStrictFP())
1268 auto IID = SrcCI->getIntrinsicID();
1273 if (IID == Intrinsic::amdgcn_sqrt || IID == Intrinsic::sqrt) {
1283 SrcCI->getModule(), Intrinsic::amdgcn_rsq, {SrcCI->getType()});
1286 II.setFastMathFlags(InnerFMF);
1288 II.setCalledFunction(NewDecl);
1294 case Intrinsic::amdgcn_sqrt:
1295 case Intrinsic::amdgcn_rsq:
1296 case Intrinsic::amdgcn_tanh: {
1297 Value *Src =
II.getArgOperand(0);
1309 if (IID == Intrinsic::amdgcn_sqrt && Src->getType()->isHalfTy()) {
1311 II.getModule(), Intrinsic::sqrt, {II.getType()});
1312 II.setCalledFunction(NewDecl);
1318 case Intrinsic::amdgcn_log:
1319 case Intrinsic::amdgcn_exp2: {
1320 const bool IsLog = IID == Intrinsic::amdgcn_log;
1321 const bool IsExp = IID == Intrinsic::amdgcn_exp2;
1322 Value *Src =
II.getArgOperand(0);
1332 if (
C->isInfinity()) {
1335 if (!
C->isNegative())
1339 if (IsExp &&
C->isNegative())
1343 if (
II.isStrictFP())
1347 Constant *Quieted = ConstantFP::get(Ty,
C->getValue().makeQuiet());
1352 if (
C->isZero() || (
C->getValue().isDenormal() && Ty->isFloatTy())) {
1354 : ConstantFP::get(Ty, 1.0);
1358 if (IsLog &&
C->isNegative())
1366 case Intrinsic::amdgcn_frexp_mant:
1367 case Intrinsic::amdgcn_frexp_exp: {
1368 Value *Src =
II.getArgOperand(0);
1374 if (IID == Intrinsic::amdgcn_frexp_mant) {
1376 II, ConstantFP::get(
II.getContext(), Significand));
1396 case Intrinsic::amdgcn_class: {
1397 Value *Src0 =
II.getArgOperand(0);
1398 Value *Src1 =
II.getArgOperand(1);
1402 II.getModule(), Intrinsic::is_fpclass, Src0->getType()));
1405 II.setArgOperand(1, ConstantInt::get(Src1->getType(),
1426 case Intrinsic::amdgcn_cvt_pkrtz: {
1427 auto foldFPTruncToF16RTZ = [](
Value *Arg) ->
Value * {
1440 return ConstantFP::get(HalfTy, Val);
1443 Value *Src =
nullptr;
1445 if (Src->getType()->isHalfTy())
1452 if (
Value *Src0 = foldFPTruncToF16RTZ(
II.getArgOperand(0))) {
1453 if (
Value *Src1 = foldFPTruncToF16RTZ(
II.getArgOperand(1))) {
1463 case Intrinsic::amdgcn_cvt_pknorm_i16:
1464 case Intrinsic::amdgcn_cvt_pknorm_u16:
1465 case Intrinsic::amdgcn_cvt_pk_i16:
1466 case Intrinsic::amdgcn_cvt_pk_u16: {
1467 Value *Src0 =
II.getArgOperand(0);
1468 Value *Src1 =
II.getArgOperand(1);
1480 case Intrinsic::amdgcn_cvt_off_f32_i4: {
1481 Value* Arg =
II.getArgOperand(0);
1495 constexpr size_t ResValsSize = 16;
1496 static constexpr float ResVals[ResValsSize] = {
1497 0.0, 0.0625, 0.125, 0.1875, 0.25, 0.3125, 0.375, 0.4375,
1498 -0.5, -0.4375, -0.375, -0.3125, -0.25, -0.1875, -0.125, -0.0625};
1500 ConstantFP::get(Ty, ResVals[CArg->
getZExtValue() & (ResValsSize - 1)]);
1503 case Intrinsic::amdgcn_ubfe:
1504 case Intrinsic::amdgcn_sbfe: {
1506 Value *Src =
II.getArgOperand(0);
1513 unsigned IntSize = Ty->getIntegerBitWidth();
1518 if ((Width & (IntSize - 1)) == 0) {
1523 if (Width >= IntSize) {
1525 II, 2, ConstantInt::get(CWidth->
getType(), Width & (IntSize - 1)));
1536 ConstantInt::get(COffset->
getType(),
Offset & (IntSize - 1)));
1540 bool Signed = IID == Intrinsic::amdgcn_sbfe;
1542 if (!CWidth || !COffset)
1552 if (
Offset + Width < IntSize) {
1556 RightShift->takeName(&
II);
1563 RightShift->takeName(&
II);
1566 case Intrinsic::amdgcn_exp:
1567 case Intrinsic::amdgcn_exp_row:
1568 case Intrinsic::amdgcn_exp_compr: {
1574 bool IsCompr = IID == Intrinsic::amdgcn_exp_compr;
1576 for (
int I = 0;
I < (IsCompr ? 2 : 4); ++
I) {
1577 if ((!IsCompr && (EnBits & (1 <<
I)) == 0) ||
1578 (IsCompr && ((EnBits & (0x3 << (2 *
I))) == 0))) {
1579 Value *Src =
II.getArgOperand(
I + 2);
1593 case Intrinsic::amdgcn_fmed3: {
1594 Value *Src0 =
II.getArgOperand(0);
1595 Value *Src1 =
II.getArgOperand(1);
1596 Value *Src2 =
II.getArgOperand(2);
1598 for (
Value *Src : {Src0, Src1, Src2}) {
1603 if (
II.isStrictFP())
1640 const APFloat *ConstSrc0 =
nullptr;
1641 const APFloat *ConstSrc1 =
nullptr;
1642 const APFloat *ConstSrc2 =
nullptr;
1647 const bool IsPosInfinity = ConstSrc0 && ConstSrc0->
isPosInfinity();
1667 const bool IsPosInfinity = ConstSrc1 && ConstSrc1->
isPosInfinity();
1690 auto *Quieted = ConstantFP::get(
II.getType(), ConstSrc2->
makeQuiet());
1710 CI->copyFastMathFlags(&
II);
1736 II.setArgOperand(0, Src0);
1737 II.setArgOperand(1, Src1);
1738 II.setArgOperand(2, Src2);
1748 ConstantFP::get(
II.getType(), Result));
1753 if (!ST->hasMed3_16())
1762 IID, {
X->getType()}, {
X,
Y, Z}, &
II,
II.getName());
1770 case Intrinsic::amdgcn_mbcnt_hi:
1775 case Intrinsic::amdgcn_mbcnt_lo: {
1785 IID == Intrinsic::amdgcn_mbcnt_lo && ST->isWave64() ? 32 : 31;
1791 if (std::optional<ConstantRange> ExistingRange =
II.getRange()) {
1792 ComputedRange = ComputedRange.
intersectWith(*ExistingRange);
1793 if (ComputedRange == *ExistingRange)
1797 II.addRangeRetAttr(ComputedRange);
1800 case Intrinsic::amdgcn_ballot: {
1801 Value *Arg =
II.getArgOperand(0);
1806 if (Src->isZero()) {
1811 if (ST->isWave32() &&
II.getType()->getIntegerBitWidth() == 64) {
1818 {IC.Builder.getInt32Ty()},
1819 {II.getArgOperand(0)}),
1826 case Intrinsic::amdgcn_wavefrontsize: {
1827 if (ST->isWaveSizeKnown())
1829 II, ConstantInt::get(
II.getType(), ST->getWavefrontSize()));
1832 case Intrinsic::amdgcn_wqm_vote: {
1839 case Intrinsic::amdgcn_kill: {
1841 if (!
C || !
C->getZExtValue())
1847 case Intrinsic::amdgcn_s_sendmsg:
1848 case Intrinsic::amdgcn_s_sendmsghalt: {
1854 Value *M0Val =
II.getArgOperand(1);
1859 uint16_t MsgId, OpId, StreamId;
1860 decodeMsg(
static_cast<unsigned>(MsgImm->getZExtValue()), MsgId, OpId,
1863 if (!msgDoesNotUseM0(MsgId, *ST))
1867 II.dropUBImplyingAttrsAndMetadata();
1871 case Intrinsic::amdgcn_update_dpp: {
1872 Value *Old =
II.getArgOperand(0);
1877 if (BC->isNullValue() || RM->getZExtValue() != 0xF ||
1884 case Intrinsic::amdgcn_permlane16:
1885 case Intrinsic::amdgcn_permlane16_var:
1886 case Intrinsic::amdgcn_permlanex16:
1887 case Intrinsic::amdgcn_permlanex16_var: {
1889 Value *VDstIn =
II.getArgOperand(0);
1894 unsigned int FiIdx = (IID == Intrinsic::amdgcn_permlane16 ||
1895 IID == Intrinsic::amdgcn_permlanex16)
1902 unsigned int BcIdx = FiIdx + 1;
1911 case Intrinsic::amdgcn_wave_shuffle:
1913 case Intrinsic::amdgcn_permlane64:
1914 case Intrinsic::amdgcn_readfirstlane:
1915 case Intrinsic::amdgcn_readlane:
1916 case Intrinsic::amdgcn_ds_bpermute: {
1918 unsigned SrcIdx = IID == Intrinsic::amdgcn_ds_bpermute ? 1 : 0;
1919 const Use &Src =
II.getArgOperandUse(SrcIdx);
1923 if (IID == Intrinsic::amdgcn_readlane &&
1930 if (IID == Intrinsic::amdgcn_ds_bpermute) {
1931 const Use &Lane =
II.getArgOperandUse(0);
1935 II.getModule(), Intrinsic::amdgcn_readlane,
II.getType());
1936 II.setCalledFunction(NewDecl);
1937 II.setOperand(0, Src);
1938 II.setOperand(1, NewLane);
1943 if (IID == Intrinsic::amdgcn_ds_bpermute)
1949 return std::nullopt;
1951 case Intrinsic::amdgcn_wave_match_b32: {
1952 const Use &Src0 =
II.getArgOperandUse(0);
1953 const Use &Src1 =
II.getArgOperandUse(1);
1956 II.getModule(), Intrinsic::amdgcn_ballot,
II.getType());
1963 case Intrinsic::amdgcn_writelane: {
1967 return std::nullopt;
1969 case Intrinsic::amdgcn_trig_preop: {
1972 if (!
II.getType()->isDoubleTy())
1975 Value *Src =
II.getArgOperand(0);
1976 Value *Segment =
II.getArgOperand(1);
1985 if (StrippedSign != Src)
1988 if (
II.isStrictFP())
2010 unsigned SegmentVal =
2011 static_cast<unsigned>(Cseg->
getValue().trunc(5).getZExtValue());
2012 unsigned Shift = SegmentVal * 53;
2017 static const uint32_t TwoByPi[] = {
2018 0xa2f9836e, 0x4e441529, 0xfc2757d1, 0xf534ddc0, 0xdb629599, 0x3c439041,
2019 0xfe5163ab, 0xdebbc561, 0xb7246e3a, 0x424dd2e0, 0x06492eea, 0x09d1921c,
2020 0xfe1deb1c, 0xb129a73e, 0xe88235f5, 0x2ebb4484, 0xe99c7026, 0xb45f7e41,
2021 0x3991d639, 0x835339f4, 0x9c845f8b, 0xbdf9283b, 0x1ff897ff, 0xde05980f,
2022 0xef2f118b, 0x5a0a6d1f, 0x6d367ecf, 0x27cb09b7, 0x4f463f66, 0x9e5fea2d,
2023 0x7527bac7, 0xebe5f17b, 0x3d0739f7, 0x8a5292ea, 0x6bfb5fb1, 0x1f8d5d08,
2027 unsigned Idx = Shift >> 5;
2028 if (Idx + 2 >= std::size(TwoByPi)) {
2033 unsigned BShift = Shift & 0x1f;
2034 uint64_t Thi =
Make_64(TwoByPi[Idx], TwoByPi[Idx + 1]);
2035 uint64_t Tlo =
Make_64(TwoByPi[Idx + 2], 0);
2037 Thi = (Thi << BShift) | (Tlo >> (64 - BShift));
2041 int Scale = -53 - Shift;
2048 case Intrinsic::amdgcn_sdot2:
2049 case Intrinsic::amdgcn_udot2:
2050 case Intrinsic::amdgcn_sdot4:
2051 case Intrinsic::amdgcn_udot4:
2052 case Intrinsic::amdgcn_sdot8:
2053 case Intrinsic::amdgcn_udot8: {
2054 Value *Src0 =
II.getArgOperand(0);
2055 Value *Src1 =
II.getArgOperand(1);
2059 II.setArgOperand(0, Src1);
2060 II.setArgOperand(1, Src0);
2072 case Intrinsic::amdgcn_sudot4:
2073 case Intrinsic::amdgcn_sudot8: {
2074 Value *Src0 =
II.getArgOperand(1);
2075 Value *Src1 =
II.getArgOperand(3);
2080 Value *Sign0 =
II.getArgOperand(0);
2081 Value *Sign1 =
II.getArgOperand(2);
2082 II.setArgOperand(0, Sign1);
2083 II.setArgOperand(1, Src1);
2084 II.setArgOperand(2, Sign0);
2085 II.setArgOperand(3, Src0);
2097 case Intrinsic::amdgcn_fmul_legacy: {
2098 Value *Op0 =
II.getArgOperand(0);
2099 Value *Op1 =
II.getArgOperand(1);
2101 for (
Value *Src : {Op0, Op1}) {
2122 case Intrinsic::amdgcn_fma_legacy: {
2123 Value *Op0 =
II.getArgOperand(0);
2124 Value *Op1 =
II.getArgOperand(1);
2125 Value *Op2 =
II.getArgOperand(2);
2127 for (
Value *Src : {Op0, Op1, Op2}) {
2149 II.getModule(), Intrinsic::fma,
II.getType()));
2154 case Intrinsic::amdgcn_is_shared:
2155 case Intrinsic::amdgcn_is_private: {
2156 Value *Src =
II.getArgOperand(0);
2166 case Intrinsic::amdgcn_make_buffer_rsrc: {
2167 Value *Src =
II.getArgOperand(0);
2172 std::optional<unsigned> Width = ST->getBufferResourceNumRecordsWidth();
2174 return std::nullopt;
2176 if (
II.getArgOperand(2)->getType() == NumRecordsTy)
2177 return std::nullopt;
2181 Intrinsic::amdgcn_make_buffer_rsrc,
2182 {
II.getType(), Src->getType(), NumRecordsTy}, Args);
2188 II.getAttributes().removeParamAttributes(
II.getContext(), 2));
2192 case Intrinsic::amdgcn_raw_buffer_store_format:
2193 case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
2194 case Intrinsic::amdgcn_struct_buffer_store_format:
2195 case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
2196 case Intrinsic::amdgcn_raw_tbuffer_store:
2197 case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
2198 case Intrinsic::amdgcn_struct_tbuffer_store:
2199 case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
2200 case Intrinsic::amdgcn_image_store_1d:
2201 case Intrinsic::amdgcn_image_store_1darray:
2202 case Intrinsic::amdgcn_image_store_2d:
2203 case Intrinsic::amdgcn_image_store_2darray:
2204 case Intrinsic::amdgcn_image_store_2darraymsaa:
2205 case Intrinsic::amdgcn_image_store_2dmsaa:
2206 case Intrinsic::amdgcn_image_store_3d:
2207 case Intrinsic::amdgcn_image_store_cube:
2208 case Intrinsic::amdgcn_image_store_mip_1d:
2209 case Intrinsic::amdgcn_image_store_mip_1darray:
2210 case Intrinsic::amdgcn_image_store_mip_2d:
2211 case Intrinsic::amdgcn_image_store_mip_2darray:
2212 case Intrinsic::amdgcn_image_store_mip_3d:
2213 case Intrinsic::amdgcn_image_store_mip_cube: {
2218 if (ST->hasDefaultComponentBroadcast())
2220 else if (ST->hasDefaultComponentZero())
2225 int DMaskIdx = getAMDGPUImageDMaskIntrinsic(
II.getIntrinsicID()) ? 1 : -1;
2233 case Intrinsic::amdgcn_prng_b32: {
2234 auto *Src =
II.getArgOperand(0);
2238 return std::nullopt;
2240 case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4:
2241 case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: {
2242 Value *Src0 =
II.getArgOperand(0);
2243 Value *Src1 =
II.getArgOperand(1);
2249 auto getFormatNumRegs = [](uint64_t FormatVal) {
2250 switch (FormatVal) {
2264 bool MadeChange =
false;
2265 unsigned Src0NumElts = getFormatNumRegs(CBSZ);
2266 unsigned Src1NumElts = getFormatNumRegs(BLGP);
2270 if (Src0Ty->getNumElements() > Src0NumElts) {
2277 if (Src1Ty->getNumElements() > Src1NumElts) {
2285 return std::nullopt;
2292 IID, {Src0->getType(), Src1->getType()}, Args, &
II);
2296 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
2297 case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4:
2298 case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: {
2299 Value *Src0 =
II.getArgOperand(1);
2300 Value *Src1 =
II.getArgOperand(3);
2306 bool MadeChange =
false;
2312 if (Src0Ty->getNumElements() > Src0NumElts) {
2319 if (Src1Ty->getNumElements() > Src1NumElts) {
2327 return std::nullopt;
2334 IID, {
II.getArgOperand(5)->getType(), Src0->getType(), Src1->getType()},
2344 if (std::optional<unsigned> ScaleArgIdx =
2349 return std::nullopt;
2363 int DMaskIdx,
bool IsLoad) {
2366 :
II.getOperand(0)->getType());
2367 unsigned VWidth = IIVTy->getNumElements();
2370 Type *EltTy = IIVTy->getElementType();
2383 const unsigned UnusedComponentsAtFront = DemandedElts.
countr_zero();
2388 DemandedElts = (1 << ActiveBits) - 1;
2390 if (UnusedComponentsAtFront > 0) {
2391 static const unsigned InvalidOffsetIdx = 0xf;
2395 unsigned FormatBitsPerComp = 0;
2396 switch (
II.getIntrinsicID()) {
2397 case Intrinsic::amdgcn_raw_buffer_load:
2398 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2401 case Intrinsic::amdgcn_s_buffer_load:
2402 case Intrinsic::amdgcn_ptr_s_buffer_load:
2406 if (ActiveBits == 4 && UnusedComponentsAtFront == 1)
2407 OffsetIdx = InvalidOffsetIdx;
2411 case Intrinsic::amdgcn_struct_buffer_load:
2412 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2415 case Intrinsic::amdgcn_raw_tbuffer_load:
2416 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2417 case Intrinsic::amdgcn_struct_tbuffer_load:
2418 case Intrinsic::amdgcn_struct_ptr_tbuffer_load: {
2420 const bool HasVIndex =
2421 II.getIntrinsicID() == Intrinsic::amdgcn_struct_tbuffer_load ||
2422 II.getIntrinsicID() == Intrinsic::amdgcn_struct_ptr_tbuffer_load;
2423 OffsetIdx = HasVIndex ? 2 : 1;
2424 const unsigned FormatIdx = OffsetIdx + 2;
2434 if (OldFormat && ActiveBits <= OldFormat->NumComponents)
2440 OffsetIdx = InvalidOffsetIdx;
2444 ConstantInt::get(Args[FormatIdx]->
getType(), NewFormat->
Format);
2451 OffsetIdx = InvalidOffsetIdx;
2455 if (OffsetIdx != InvalidOffsetIdx) {
2457 DemandedElts &= ~((1 << UnusedComponentsAtFront) - 1);
2458 auto *
Offset = Args[OffsetIdx];
2459 uint64_t SingleComponentSizeInBits =
2463 OffsetAdd = UnusedComponentsAtFront * SingleComponentSizeInBits / 8;
2464 auto *OffsetAddVal = ConstantInt::get(
Offset->getType(), OffsetAdd);
2484 unsigned NewDMaskVal = 0;
2485 unsigned OrigLdStIdx = 0;
2486 for (
unsigned SrcIdx = 0; SrcIdx < 4; ++SrcIdx) {
2487 const unsigned Bit = 1 << SrcIdx;
2488 if (!!(DMaskVal & Bit)) {
2489 if (!!DemandedElts[OrigLdStIdx])
2495 if (DMaskVal != NewDMaskVal)
2496 Args[DMaskIdx] = ConstantInt::get(DMask->
getType(), NewDMaskVal);
2499 unsigned NewNumElts = DemandedElts.
popcount();
2503 if (NewNumElts >= VWidth && DemandedElts.
isMask()) {
2505 II.setArgOperand(DMaskIdx, Args[DMaskIdx]);
2517 OverloadTys[0] = NewTy;
2521 for (
unsigned OrigStoreIdx = 0; OrigStoreIdx < VWidth; ++OrigStoreIdx)
2522 if (DemandedElts[OrigStoreIdx])
2525 if (NewNumElts == 1)
2532 II.getIntrinsicID(), OverloadTys, Args);
2538 if (OffsetAdd != 0) {
2551 if (NewNumElts == 1) {
2557 unsigned NewLoadIdx = 0;
2558 for (
unsigned OrigLoadIdx = 0; OrigLoadIdx < VWidth; ++OrigLoadIdx) {
2559 if (!!DemandedElts[OrigLoadIdx])
2575 APInt &UndefElts)
const {
2580 const unsigned FirstElt = DemandedElts.
countr_zero();
2582 const unsigned MaskLen = LastElt - FirstElt + 1;
2584 unsigned OldNumElts = VT->getNumElements();
2585 if (MaskLen == OldNumElts && MaskLen != 1)
2588 Type *EltTy = VT->getElementType();
2596 Value *Src =
II.getArgOperand(0);
2601 II.getOperandBundlesAsDefs(OpBundles);
2618 for (
unsigned I = 0;
I != MaskLen; ++
I) {
2619 if (DemandedElts[FirstElt +
I])
2620 ExtractMask[
I] = FirstElt +
I;
2629 for (
unsigned I = 0;
I != MaskLen; ++
I) {
2630 if (DemandedElts[FirstElt +
I])
2631 InsertMask[FirstElt +
I] =
I;
2643 SimplifyAndSetOp)
const {
2644 switch (
II.getIntrinsicID()) {
2645 case Intrinsic::amdgcn_readfirstlane:
2646 SimplifyAndSetOp(&
II, 0, DemandedElts, UndefElts);
2648 case Intrinsic::amdgcn_raw_buffer_load:
2649 case Intrinsic::amdgcn_raw_ptr_buffer_load:
2650 case Intrinsic::amdgcn_raw_buffer_load_format:
2651 case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
2652 case Intrinsic::amdgcn_raw_tbuffer_load:
2653 case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
2654 case Intrinsic::amdgcn_s_buffer_load:
2655 case Intrinsic::amdgcn_ptr_s_buffer_load:
2656 case Intrinsic::amdgcn_struct_buffer_load:
2657 case Intrinsic::amdgcn_struct_ptr_buffer_load:
2658 case Intrinsic::amdgcn_struct_buffer_load_format:
2659 case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
2660 case Intrinsic::amdgcn_struct_tbuffer_load:
2661 case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
2664 if (getAMDGPUImageDMaskIntrinsic(
II.getIntrinsicID())) {
2671 return std::nullopt;
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static Value * createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlane16 with the precomputed lane-select halves.
static std::optional< unsigned > matchRowSharePattern(ArrayRef< uint8_t > Ids)
Match a row-share pattern: all 16 lanes of each row read the same source lane.
static bool matchMirrorPattern(ArrayRef< uint8_t > Ids)
Match an N-lane reversal (mirror) pattern.
static bool canSafelyConvertTo16Bit(Value &V, bool IsFloat, bool AllowI16SExt=false)
static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST, SmallVectorImpl< uint8_t > &Ids, const DataLayout &DL)
Build the per-lane shuffle map by evaluating Index for every lane in the wave.
static std::optional< unsigned > matchQuadPermPattern(ArrayRef< uint8_t > Ids)
Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp QUAD_PERM control word: bits[1:0]=Ids...
static std::optional< unsigned > matchDsSwizzleRotatePattern(ArrayRef< uint8_t > Ids)
Match a GFX9+ DS_SWIZZLE rotate-mode permutation: a cyclic left-rotation of all 32 lanes within each ...
static std::optional< unsigned > matchHalfRowPermPattern(ArrayRef< uint8_t > Ids)
Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8 24-bit selector (three bits per ...
static std::optional< unsigned > matchRowXMaskPattern(ArrayRef< uint8_t > Ids)
Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J, with Mask in [1,...
static constexpr auto matchHalfRowMirrorPattern
static Value * createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo, uint32_t Hi)
Emit v_permlanex16 with the precomputed lane-select halves.
static bool isRowPattern(ArrayRef< uint8_t > Ids)
Match an N-lane row pattern: each lane in [0, N) reads from a source lane in the same N-lane row,...
static bool canContractSqrtToRsq(const FPMathOperator *SqrtOp)
Return true if it's legal to contract llvm.amdgcn.rcp(llvm.sqrt)
static CallInst * rewriteCall(IRBuilderBase &B, CallInst &Old, Function &NewCallee, ArrayRef< Value * > Ops)
static Value * convertTo16Bit(Value &V, InstCombiner::BuilderTy &Builder)
static constexpr auto isFullRowPattern
static constexpr auto isQuadPattern
static APInt trimTrailingZerosInVector(InstCombiner &IC, Value *UseV, Instruction *I)
static uint64_t computePermlane16Masks(ArrayRef< uint8_t > Ids)
Pack a 16-lane permutation into a single 64-bit value: four bits per output lane, lane J in bits [J*4...
static bool matchHalfWaveSwapPattern(ArrayRef< uint8_t > Ids)
Match a half-wave swap: lane J reads from lane J ^ 32.
static bool hasPeriodicLayout(ArrayRef< uint8_t > Ids)
Lanes are partitioned into groups of Period; each group is a translated copy of the first: Ids[I] = I...
static std::optional< uint64_t > evalLaneExpr(Value *V, unsigned Lane, const GCNSubtarget &ST, const DataLayout &DL, unsigned Depth=0)
Evaluate V as a function of the lane ID and return its value on Lane, or std::nullopt if V is not a c...
static std::optional< Instruction * > tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II, const GCNSubtarget &ST)
Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant function of the lane ID into a ...
static constexpr auto isHalfRowPattern
static Value * simplifyAMDGCNMemoryIntrinsicDemanded(const GCNSubtarget &ST, InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, int DMaskIdx=-1, bool IsLoad=true)
Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
static APInt defaultComponentBroadcast(Value *V)
static std::optional< unsigned > matchDsSwizzleBitmaskPattern(ArrayRef< uint8_t > Ids)
Match a DS_SWIZZLE bitmask-mode permutation: dst_lane = ((src_lane & AND) | OR) ^ XOR with each mask ...
static Value * createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset, const DataLayout &DL)
Emit ds_swizzle with the given immediate, bitcasting/converting between pointer/float types and i32 a...
static std::optional< Instruction * > modifyIntrinsicCall(IntrinsicInst &OldIntr, Instruction &InstToReplace, unsigned NewIntr, InstCombiner &IC, std::function< void(SmallVectorImpl< Value * > &, SmallVectorImpl< Type * > &)> Func)
Applies Func(OldIntr.Args, OldIntr.ArgTys), creates intrinsic call with modified arguments (based on ...
static Value * matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src, ArrayRef< uint8_t > Ids, const GCNSubtarget &ST, const DataLayout &DL)
Given a shuffle map, try to emit the best hardware intrinsic.
static std::optional< unsigned > matchRowRotatePattern(ArrayRef< uint8_t > Ids)
Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
static bool isCrossRowPattern(ArrayRef< uint8_t > Ids)
Match a cross-row permutation suitable for v_permlanex16: every lane in the low 16-lane half reads fr...
static bool isThreadID(const GCNSubtarget &ST, Value *V)
static Value * createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl)
Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and bound_ctrl=1 so out-of-bounds...
static APFloat fmed3AMDGCN(const APFloat &Src0, const APFloat &Src1, const APFloat &Src2)
static std::optional< Instruction * > simplifyAMDGCNImageIntrinsic(const GCNSubtarget *ST, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr, IntrinsicInst &II, InstCombiner &IC)
static Value * createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector)
Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
static Instruction * foldAddIntoDotAccumulator(IntrinsicInst &II, unsigned AccIdx, unsigned ClampIdx, InstCombiner &IC)
Try to fold an addition into the accumulator when saturation is disabled.
static Value * matchFPExtFromF16(Value *Arg)
Match an fpext from half to float, or a constant we can convert.
static constexpr auto matchFullRowMirrorPattern
static bool simplifyExponentOnlyScaleArg(InstCombiner &IC, IntrinsicInst &II, unsigned ScaleArgIdx)
Simplify the f32 scale operand of a V_CVT_SCALEF32_* conversion, of which only the exponent field is ...
static Value * createPermlane64(IRBuilderBase &B, Value *Val)
Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Utilities for dealing with flags related to floating point properties and mode controls.
AMD GCN specific subclass of TargetSubtarget.
This file provides the interface for the instcombine pass implementation.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
uint64_t IntrinsicInst * II
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static SymbolRef::Type getType(const Symbol *Sym)
static const fltSemantics & IEEEsingle()
static constexpr roundingMode rmTowardZero
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
bool bitwiseIsEqual(const APFloat &RHS) const
bool isPosInfinity() const
APFloat makeQuiet() const
Assuming this is an IEEE-754 NaN value, quiet its signaling bit.
APInt bitcastToAPInt() const
static APFloat getZero(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Zero.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
unsigned popcount() const
Count the number of bits set.
LLVM_ABI uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isMask(unsigned numBits) const
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
size_t size() const
Get the array size.
static LLVM_ABI Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
bool isTypeLegal(Type *Ty) const override
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
void setAttributes(AttributeList A)
Set the attributes for this call.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
AttributeList getAttributes() const
Return the attributes for this call.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
static LLVM_ABI ConstantFP * getZero(Type *Ty, bool Negative=false)
static LLVM_ABI ConstantFP * getNaN(Type *Ty, bool Negative=false, uint64_t Payload=0)
static LLVM_ABI ConstantFP * getInfinity(Type *Ty, bool Negative=false)
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
LLVM_ABI ConstantRange add(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an addition of a value in this ran...
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI ConstantRange intersectWith(const ConstantRange &CR, PreferredRangeType Type=Smallest) const
Return the range that results from the intersection of this range with another range.
This is an important base class in LLVM.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Tagged union holding either a T or a Error.
This class represents an extension of floating point types.
Utility class for floating point operations which can have information about relaxed accuracy require...
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
bool hasApproxFunc() const
Test if this operation allows approximations of math library functions or intrinsics.
LLVM_ABI float getFPAccuracy() const
Get the maximum error permitted by this operation in ULPs.
Convenience struct for specifying and reasoning about fast-math flags.
bool allowContract() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II, unsigned LaneAgIdx) const
Simplify a lane index operand (e.g.
std::optional< Instruction * > instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override
Instruction * hoistLaneIntrinsicThroughOperand(InstCombiner &IC, IntrinsicInst &II) const
std::optional< Value * > simplifyDemandedVectorEltsIntrinsic(InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3, std::function< void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) const override
KnownIEEEMode fpenvIEEEMode(const Instruction &I) const
Return KnownIEEEMode::On if we know if the use context can assume "amdgpu-ieee"="true" and KnownIEEEM...
Value * simplifyAMDGCNLaneIntrinsicDemanded(InstCombiner &IC, IntrinsicInst &II, const APInt &DemandedElts, APInt &UndefElts) const
bool canSimplifyLegacyMulToMul(const Instruction &I, const Value *Op0, const Value *Op1, InstCombiner &IC) const
Common base class shared among various IRBuilders.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
LLVM_ABI CallInst * CreateIntrinsicWithoutFolding(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={})
Create a call to intrinsic ID with Args, mangled using OverloadTypes.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name="")
Create a call to the vector.extract intrinsic.
Module * getModule() const
Get the module.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateMaxNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the maxnum intrinsic.
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
Value * CreateMaximumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the maximum intrinsic.
Value * CreateMinNum(Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create call to the minnum intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateFAddFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateMinimumNum(Value *LHS, Value *RHS, const Twine &Name="")
Create call to the minimumnum intrinsic.
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateFMulFMF(Value *L, Value *R, FMFSource FMFSource, const Twine &Name="", MDNode *FPMD=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
The core instruction combiner logic.
const DataLayout & getDataLayout() const
virtual Instruction * eraseInstFromFunction(Instruction &I)=0
Combiner aware instruction erasure.
DominatorTree & getDominatorTree() const
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
virtual bool SimplifyDemandedBits(Instruction *I, unsigned OpNo, const APInt &DemandedMask, KnownBits &Known, const SimplifyQuery &Q, unsigned Depth=0)=0
IRBuilder< TargetFolder, IRBuilderInstCombineInserter > BuilderTy
An IRBuilder that automatically inserts new instructions into the worklist.
static Value * stripSignOnlyFPOps(Value *Val)
Ignore all operations which only change the sign of a value, returning the underlying magnitude value...
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
const SimplifyQuery & getSimplifyQuery() const
LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef< unsigned > Keep={})
Drop any attributes or metadata that can cause immediate undefined behavior.
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
void dropPoisonGeneratingAnnotations()
Drops flags, attributes and metadata that may generate poison.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
bool isKnownNeverInfOrNaN() const
Return true if it's known this can never be an infinity or nan.
LLVM_ABI bool isKnownNeverLogicalZero(DenormalMode Mode) const
Return true if it's known this can never be interpreted as a zero.
A Module instance is used to store all the information related to an LLVM module.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI Type * getHalfTy(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
const Use & getOperandUse(unsigned i) const
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
constexpr ScalarTy getFixedValue() const
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt getExponentOnlyScaleDemandedBits()
LLVM_READONLY const MIMGOffsetMappingInfo * getMIMGOffsetMappingInfo(unsigned Offset)
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const ImageDimIntrinsicInfo * getImageDimIntrinsicByBaseOpcode(unsigned BaseOpcode, unsigned Dim)
std::optional< unsigned > getExponentOnlyScaleArgIdx(unsigned IntrID)
LLVM_READONLY const MIMGMIPMappingInfo * getMIMGMIPMappingInfo(unsigned MIP)
bool isTriviallyUniform(const Value *V, const Instruction *CtxI)
Return true if we can easily prove that V is uniform at CtxI.
LLVM_READONLY const MIMGBiasMappingInfo * getMIMGBiasMappingInfo(unsigned Bias)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
LLVM_READONLY const MIMGLZMappingInfo * getMIMGLZMappingInfo(unsigned L)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfo(unsigned DimEnum)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
const ImageDimIntrinsicInfo * getImageDimIntrinsicInfo(unsigned Intr)
const RsrcIntrinsic * lookupRsrcIntrinsic(unsigned Intr)
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI bool isSignatureValid(Intrinsic::ID ID, FunctionType *FT, SmallVectorImpl< Type * > &OverloadTys, raw_ostream &OS=nulls())
Returns true if FT is a valid function type for intrinsic ID.
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_bind< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
cstfp_pred_ty< is_any_zero_fp > m_AnyZeroFP()
Match a floating-point negative zero or positive zero.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ap_match< APFloat > m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
auto m_Value()
Match an arbitrary value and ignore it.
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
is_zero< true > m_Zero()
Match any null constant or a vector with all elements equal to 0.
BinaryOp_match< LHS, RHS, Instruction::Add, true > m_c_Add(const LHS &L, const RHS &R)
Matches a Add with LHS and RHS in either order.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_ConstantFP()
Match an arbitrary ConstantFP and ignore it.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
@ Known
Known to have no common set bits.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
constexpr bool isMask_32(uint32_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
auto dyn_cast_or_null(const Y &Val)
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2008 maxNum semantics.
constexpr unsigned MaxAnalysisRecursionDepth
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
constexpr int PoisonMaskElem
LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
@ NearestTiesToEven
roundTiesToEven.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
LLVM_ABI Constant * ConstantFoldInstOperands(const Instruction *I, ArrayRef< Constant * > Ops, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, bool AllowNonDeterministic=true)
ConstantFoldInstOperands - Attempt to constant fold an instruction with the specified operands.
constexpr uint64_t Make_64(uint32_t High, uint32_t Low)
Make a 64-bit integer from a high / low pair of 32-bit integers.
LLVM_ABI ConstantRange computeConstantRange(const Value *V, bool ForSigned, const SimplifyQuery &SQ, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Represent subnormal handling kind for floating point instruction inputs and outputs.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
SimplifyQuery getWithInstruction(const Instruction *I) const
LLVM_ABI bool isUndefValue(Value *V) const
If CanUseUndef is true, returns whether V is undef.