24#include "llvm/IR/IntrinsicsDirectX.h"
34#define DEBUG_TYPE "dxil-intrinsic-expansion"
49 if (IsRaw && M->getTargetTriple().getDXILVersion() >
VersionTuple(1, 2))
58 if (M->getTargetTriple().getDXILVersion() >=
VersionTuple(1, 9))
73 ConstantInt::get(IType, 0x7c00))
74 : ConstantInt::get(IType, 0x7c00);
81 ConstantInt::get(IType, 0xfc00))
82 : ConstantInt::get(IType, 0xfc00);
84 Value *IVal = Builder.CreateBitCast(Val, PosInf->
getType());
85 Value *B1 = Builder.CreateICmpEQ(IVal, PosInf);
86 Value *B2 = Builder.CreateICmpEQ(IVal, NegInf);
87 Value *B3 = Builder.CreateOr(B1, B2);
93 if (M->getTargetTriple().getDXILVersion() >=
VersionTuple(1, 9))
109 ConstantInt::get(IType, 0x7c00))
110 : ConstantInt::get(IType, 0x7c00);
116 ConstantInt::get(IType, 0x3ff))
117 : ConstantInt::get(IType, 0x3ff);
124 ConstantInt::get(IType, 0))
125 : ConstantInt::get(IType, 0);
127 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->
getType());
128 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);
129 Value *B1 = Builder.CreateICmpEQ(Exp, ExpBitMask);
131 Value *Sig = Builder.CreateAnd(IVal, SigBitMask);
132 Value *B2 = Builder.CreateICmpNE(Sig,
Zero);
133 Value *B3 = Builder.CreateAnd(B1, B2);
139 if (M->getTargetTriple().getDXILVersion() >=
VersionTuple(1, 9))
155 ConstantInt::get(IType, 0x7c00))
156 : ConstantInt::get(IType, 0x7c00);
158 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->
getType());
159 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);
160 Value *B1 = Builder.CreateICmpNE(Exp, ExpBitMask);
166 if (M->getTargetTriple().getDXILVersion() >=
VersionTuple(1, 9))
182 ConstantInt::get(IType, 0x7c00))
183 : ConstantInt::get(IType, 0x7c00);
189 ConstantInt::get(IType, 0))
190 : ConstantInt::get(IType, 0);
192 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->
getType());
193 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);
194 Value *NotAllZeroes = Builder.CreateICmpNE(Exp,
Zero);
195 Value *NotAllOnes = Builder.CreateICmpNE(Exp, ExpBitMask);
196 Value *B1 = Builder.CreateAnd(NotAllZeroes, NotAllOnes);
201 assert(
F.getIntrinsicID() == Intrinsic::dx_fdot &&
202 "Function is not a dx.fdot intrinsic");
204 return ParamTy->getNumElements() <= 4 ||
205 F.getParent()->getTargetTriple().getOSVersion() <
VersionTuple(6, 9);
209 switch (
F.getIntrinsicID()) {
210 case Intrinsic::assume:
212 case Intrinsic::atan2:
213 case Intrinsic::copysign:
214 case Intrinsic::fshl:
215 case Intrinsic::fshr:
217 case Intrinsic::is_fpclass:
219 case Intrinsic::log10:
221 case Intrinsic::powi:
222 case Intrinsic::dx_uclamp:
223 case Intrinsic::dx_sclamp:
224 case Intrinsic::dx_nclamp:
225 case Intrinsic::dx_isfinite:
226 case Intrinsic::dx_isinf:
227 case Intrinsic::dx_isnan:
228 case Intrinsic::dx_sdot:
229 case Intrinsic::dx_udot:
230 case Intrinsic::dx_sign:
231 case Intrinsic::usub_sat:
232 case Intrinsic::vector_reduce_add:
233 case Intrinsic::vector_reduce_fadd:
234 case Intrinsic::matrix_multiply:
235 case Intrinsic::matrix_transpose:
236 case Intrinsic::umul_with_overflow:
237 case Intrinsic::smul_with_overflow:
238 case Intrinsic::dx_load_input:
239 case Intrinsic::dx_store_output:
241 case Intrinsic::vector_reduce_and:
242 case Intrinsic::vector_reduce_or:
243 return F.getParent()->getTargetTriple().getDXILVersion() <
245 case Intrinsic::dx_quad_all:
246 return F.getParent()->getTargetTriple().getDXILVersion() <
248 case Intrinsic::dx_fdot:
250 case Intrinsic::dx_resource_load_rawbuffer:
252 F.getParent(),
F.getReturnType()->getStructElementType(0),
254 case Intrinsic::dx_resource_load_typedbuffer:
256 F.getParent(),
F.getReturnType()->getStructElementType(0),
258 case Intrinsic::dx_resource_store_rawbuffer:
260 F.getParent(),
F.getFunctionType()->getParamType(3),
true);
261 case Intrinsic::dx_resource_store_typedbuffer:
263 F.getParent(),
F.getFunctionType()->getParamType(2),
false);
271 Type *Ty =
A->getType();
275 Value *Cmp = Builder.CreateICmpULT(
A,
B,
"usub.cmp");
276 Value *
Sub = Builder.CreateSub(
A,
B,
"usub.sub");
278 return Builder.CreateSelect(Cmp,
Zero,
Sub,
"usub.sat");
285 Type *Ty,
unsigned BW) {
286 assert(BW % 2 == 0 &&
"high-half split needs symmetric halves");
287 unsigned Half = BW / 2;
288 Value *HalfShift = ConstantInt::get(Ty, Half);
291 Value *U0 = Builder.CreateAnd(
A, LoMask);
292 Value *U1 = Builder.CreateLShr(
A, HalfShift);
293 Value *V0 = Builder.CreateAnd(
B, LoMask);
294 Value *
V1 = Builder.CreateLShr(
B, HalfShift);
296 Value *W0 = Builder.CreateMul(U0, V0);
297 Value *
T = Builder.CreateAdd(Builder.CreateMul(U1, V0),
298 Builder.CreateLShr(W0, HalfShift));
299 Value *W1 = Builder.CreateAnd(
T, LoMask);
300 Value *W2 = Builder.CreateLShr(
T, HalfShift);
301 W1 = Builder.CreateAdd(Builder.CreateMul(U0,
V1), W1);
302 return Builder.CreateAdd(Builder.CreateAdd(Builder.CreateMul(U1,
V1), W2),
303 Builder.CreateLShr(W1, HalfShift));
313 Type *Ty =
A->getType();
314 unsigned BW = Ty->getScalarSizeInBits();
324 Lo = Builder.CreateMul(
A,
B);
327 Signed ? Builder.CreateSExt(
A, WideTy) : Builder.CreateZExt(
A, WideTy);
329 Signed ? Builder.CreateSExt(
B, WideTy) : Builder.CreateZExt(
B, WideTy);
330 Value *Wide = Builder.CreateMul(WideA, WideB);
333 Ov = Builder.CreateICmpNE(Wide, Builder.CreateSExt(
Lo, WideTy));
335 Value *
Hi = Builder.CreateLShr(Wide, ConstantInt::get(WideTy, BW));
336 Ov = Builder.CreateICmpNE(
Hi, ConstantInt::get(WideTy, 0));
338 }
else if (BW == 32) {
342 Signed ? Intrinsic::dx_imul : Intrinsic::dx_umul;
343 Value *
Mul = Builder.CreateIntrinsic(ResTy, IntrinsicID, {
A,
B});
344 Value *
Hi = Builder.CreateExtractValue(
Mul, 0);
345 Lo = Builder.CreateExtractValue(
Mul, 1);
347 Ov = Builder.CreateICmpNE(
348 Hi, Builder.CreateAShr(
Lo, ConstantInt::get(Ty, BW - 1)));
350 Ov = Builder.CreateICmpNE(
Hi, ConstantInt::get(Ty, 0));
352 Lo = Builder.CreateMul(
A,
B);
357 Value *SignShift = ConstantInt::get(Ty, BW - 1);
358 Value *ASign = Builder.CreateAShr(
A, SignShift);
359 Value *BSign = Builder.CreateAShr(
B, SignShift);
360 Hi = Builder.CreateSub(
Hi, Builder.CreateAnd(ASign,
B));
361 Hi = Builder.CreateSub(
Hi, Builder.CreateAnd(BSign,
A));
362 Ov = Builder.CreateICmpNE(
Hi, Builder.CreateAShr(
Lo, SignShift));
364 Ov = Builder.CreateICmpNE(
Hi, ConstantInt::get(Ty, 0));
369 Agg = Builder.CreateInsertValue(Agg,
Lo, 0);
370 return Builder.CreateInsertValue(Agg, Ov, 1);
374 assert(IntrinsicId == Intrinsic::vector_reduce_add ||
375 IntrinsicId == Intrinsic::vector_reduce_fadd);
378 bool IsFAdd = (IntrinsicId == Intrinsic::vector_reduce_fadd);
381 Type *Ty =
X->getType();
383 unsigned XVecSize = XVec->getNumElements();
384 Value *Sum = Builder.CreateExtractElement(
X,
static_cast<uint64_t>(0));
390 Sum = Builder.CreateFAdd(Sum, StartValue);
394 for (
unsigned I = 1;
I < XVecSize;
I++) {
395 Value *Elt = Builder.CreateExtractElement(
X,
I);
397 Sum = Builder.CreateFAdd(Sum, Elt);
399 Sum = Builder.CreateAdd(Sum, Elt);
406 assert(IntrinsicId == Intrinsic::vector_reduce_and ||
407 IntrinsicId == Intrinsic::vector_reduce_or);
412 Value *Result = Builder.CreateExtractElement(
X,
static_cast<uint64_t>(0));
413 for (
unsigned I = 1;
I < XVec->getNumElements(); ++
I) {
414 Value *Elt = Builder.CreateExtractElement(
X,
I);
415 Result = IntrinsicId == Intrinsic::vector_reduce_or
416 ? Builder.CreateOr(Result, Elt)
417 : Builder.CreateAnd(Result, Elt);
425 Type *Ty =
X->getType();
431 ConstantInt::get(EltTy, 0))
432 : ConstantInt::get(EltTy, 0);
433 auto *
V = Builder.CreateSub(
Zero,
X);
434 return Builder.CreateIntrinsic(Ty, Intrinsic::smax, {
X,
V},
nullptr,
440 Type *ATy =
A->getType();
441 [[maybe_unused]]
Type *BTy =
B->getType();
450 unsigned NumElts = AVec->getNumElements();
454 DotIntrinsic = Intrinsic::dx_dot2;
457 DotIntrinsic = Intrinsic::dx_dot3;
460 DotIntrinsic = Intrinsic::dx_dot4;
464 "Invalid dot product input vector: length is outside 2-4");
468 for (
unsigned I = 0;
I < NumElts; ++
I)
469 Args.push_back(Builder.CreateExtractElement(
A, Builder.getInt32(
I)));
470 for (
unsigned I = 0;
I < NumElts; ++
I)
471 Args.push_back(Builder.CreateExtractElement(
B, Builder.getInt32(
I)));
472 return Builder.CreateIntrinsic(ATy->
getScalarType(), DotIntrinsic, Args,
489 "long fdot must not be expanded for shader model 6.9 or later");
492 Value *Result =
nullptr;
494 unsigned Remaining = NumElts -
Offset;
497 unsigned ChunkSize = Remaining == 5 ? 3 : std::min(Remaining, 4u);
499 for (
unsigned I = 0;
I < ChunkSize; ++
I)
501 Value *AChunk = Builder.CreateShuffleVector(
A, Mask);
502 Value *BChunk = Builder.CreateShuffleVector(
B, Mask);
504 Result = Result ? Builder.CreateFAdd(Result, Chunk,
"dot.add") : Chunk;
513 assert(DotIntrinsic == Intrinsic::dx_sdot ||
514 DotIntrinsic == Intrinsic::dx_udot);
517 Type *ATy =
A->getType();
518 [[maybe_unused]]
Type *BTy =
B->getType();
528 Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot
530 : Intrinsic::dx_umad;
533 Result = Builder.CreateMul(Elt0, Elt1);
534 for (
unsigned I = 1;
I < AVec->getNumElements();
I++) {
535 Elt0 = Builder.CreateExtractElement(
A,
I);
536 Elt1 = Builder.CreateExtractElement(
B,
I);
537 Result = Builder.CreateIntrinsic(Result->getType(), MadIntrinsic,
547 Type *Ty =
X->getType();
555 Value *NewX = Builder.CreateFMul(Log2eConst,
X);
556 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
557 Ty, Intrinsic::exp2, {NewX},
nullptr,
"dx.exp2");
569 switch (TCI->getZExtValue()) {
583 Type *FTy =
F->getType();
584 unsigned FNumElem = 0;
591 FNumElem = FVecTy->getNumElements();
599 Value *FBitCast = Builder.CreateBitCast(
F, BitCastTy);
600 switch (TCI->getZExtValue()) {
602 Value *NegZero = ConstantInt::get(Builder.getIntNTy(
BitWidth),
606 Value *NegZeroSplat = Builder.CreateVectorSplat(FNumElem, NegZero);
608 Builder.CreateICmpEQ(FBitCast, NegZeroSplat,
"is.fpclass.negzero");
610 RetVal = Builder.CreateICmpEQ(FBitCast, NegZero,
"is.fpclass.negzero");
622 Type *Ty =
X->getType();
628 ConstantFP::get(EltTy, LogConstVal))
629 : ConstantFP::get(EltTy, LogConstVal);
630 CallInst *Log2Call = Builder.CreateIntrinsicWithoutFolding(
631 Ty, Intrinsic::log2, {
X},
nullptr,
"elt.log2");
634 return Builder.CreateFMul(Ln2Const, Log2Call);
643 Type *Ty =
X->getType();
647 Value *Tan = Builder.CreateFDiv(
Y,
X);
649 CallInst *Atan = Builder.CreateIntrinsicWithoutFolding(
650 Ty, Intrinsic::atan, {Tan},
nullptr,
"Elt.Atan");
659 Value *AtanAddPi = Builder.CreateFAdd(Atan, Pi);
660 Value *AtanSubPi = Builder.CreateFSub(Atan, Pi);
663 Value *Result = Atan;
670 Value *XLt0AndYGe0 = Builder.CreateAnd(XLt0, YGe0);
671 Result = Builder.CreateSelect(XLt0AndYGe0, AtanAddPi, Result);
674 Value *XLt0AndYLt0 = Builder.CreateAnd(XLt0, YLt0);
675 Result = Builder.CreateSelect(XLt0AndYLt0, AtanSubPi, Result);
678 Value *XEq0AndYLt0 = Builder.CreateAnd(XEq0, YLt0);
679 Result = Builder.CreateSelect(XEq0AndYLt0, NegHalfPi, Result);
682 Value *XEq0AndYGe0 = Builder.CreateAnd(XEq0, YGe0);
683 Result = Builder.CreateSelect(XEq0AndYGe0, HalfPi, Result);
688template <
bool LeftFunnel>
698 "Can't use Mask to compute modulo and inverse");
713 Constant *Mask = ConstantInt::get(Ty, Ty->getScalarSizeInBits() - 1);
718 Value *MaskedShift = Builder.CreateAnd(Shift, Mask);
723 Value *NotShift = Builder.CreateNot(Shift);
724 Value *InverseShift = Builder.CreateAnd(NotShift, Mask);
726 Constant *One = ConstantInt::get(Ty, 1);
731 ShiftedA = Builder.CreateShl(
A, MaskedShift);
732 Value *ShiftB1 = Builder.CreateLShr(
B, One);
733 ShiftedB = Builder.CreateLShr(ShiftB1, InverseShift);
735 Value *ShiftA1 = Builder.CreateShl(
A, One);
736 ShiftedA = Builder.CreateShl(ShiftA1, InverseShift);
737 ShiftedB = Builder.CreateLShr(
B, MaskedShift);
740 Value *Result = Builder.CreateOr(ShiftedA, ShiftedB);
748 Type *Ty =
X->getType();
751 if (IntrinsicId == Intrinsic::powi)
752 Y = Builder.CreateSIToFP(
Y, Ty);
755 Builder.CreateIntrinsic(Ty, Intrinsic::log2, {
X},
nullptr,
"elt.log2");
756 auto *
Mul = Builder.CreateFMul(Log2Call,
Y);
757 CallInst *Exp2Call = Builder.CreateIntrinsicWithoutFolding(
758 Ty, Intrinsic::exp2, {
Mul},
nullptr,
"elt.exp2");
771 "Only expand double or int64 scalars or vectors");
772 bool IsVector =
false;
773 unsigned ExtractNum = 2;
775 ExtractNum = 2 * VT->getNumElements();
777 assert(IsRaw || ExtractNum == 4 &&
"TypedBufferLoad vector must be size 2");
786 while (ExtractNum > 0) {
787 unsigned LoadNum = std::min(ExtractNum, 4u);
791 Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer;
794 LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer;
795 Value *Tmp = Builder.getInt32(4 *
Base * 2);
800 Args[AddressArg] = Builder.CreateAdd(Args[AddressArg], Tmp);
803 Value *
Load = Builder.CreateIntrinsic(LoadType, LoadIntrinsic, Args);
807 Value *Extract = Builder.CreateExtractValue(
Load, {0});
810 for (
unsigned I = 0;
I < LoadNum; ++
I)
812 Builder.CreateExtractElement(Extract, Builder.getInt32(
I)));
815 for (
unsigned I = 0;
I < LoadNum;
I += 2) {
816 Value *Combined =
nullptr;
819 Combined = Builder.CreateIntrinsic(
820 Builder.getDoubleTy(), Intrinsic::dx_asdouble,
821 {ExtractElements[I], ExtractElements[I + 1]});
826 Builder.CreateZExt(ExtractElements[
I], Builder.getInt64Ty());
828 Builder.CreateZExt(ExtractElements[
I + 1], Builder.getInt64Ty());
830 Value *ShiftedHi = Builder.CreateShl(
Hi, Builder.getInt64(32));
832 Combined = Builder.CreateOr(
Lo, ShiftedHi);
836 Result = Builder.CreateInsertElement(Result, Combined,
837 Builder.getInt32((
I / 2) +
Base));
842 ExtractNum -= LoadNum;
846 Value *CheckBit =
nullptr;
857 if (Indices[0] == 0) {
859 EVI->replaceAllUsesWith(Result);
862 assert(Indices[0] == 1 &&
"Unexpected type for typedbufferload");
867 for (
Value *L : Loads)
868 CheckBits.
push_back(Builder.CreateExtractValue(L, {1}));
869 CheckBit = Builder.CreateAnd(CheckBits);
871 EVI->replaceAllUsesWith(CheckBit);
873 EVI->eraseFromParent();
882 unsigned ValIndex = IsRaw ? 3 : 2;
887 "Only expand double or int64 scalars or vectors");
890 bool IsVector =
false;
891 unsigned ExtractNum = 2;
894 VecLen = VT->getNumElements();
895 assert(IsRaw || VecLen == 2 &&
"TypedBufferStore vector must be size 2");
896 ExtractNum = VecLen * 2;
905 Type *SplitElementTy = Int32Ty;
909 Value *LowBits =
nullptr;
910 Value *HighBits =
nullptr;
914 Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,
916 LowBits = Builder.CreateExtractValue(Split, 0);
917 HighBits = Builder.CreateExtractValue(Split, 1);
921 Constant *ShiftAmt = Builder.getInt64(32);
927 LowBits = Builder.CreateTrunc(InputVal, SplitElementTy);
928 Value *ShiftedVal = Builder.CreateLShr(InputVal, ShiftAmt);
929 HighBits = Builder.CreateTrunc(ShiftedVal, SplitElementTy);
934 for (
unsigned I = 0;
I < VecLen; ++
I) {
936 Mask.push_back(
I + VecLen);
938 Val = Builder.CreateShuffleVector(LowBits, HighBits, Mask);
940 Val = Builder.CreateInsertElement(Val, LowBits, Builder.getInt32(0));
941 Val = Builder.CreateInsertElement(Val, HighBits, Builder.getInt32(1));
948 while (ExtractNum > 0) {
949 unsigned StoreNum = std::min(ExtractNum, 4u);
951 Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer;
954 StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer;
960 Args[AddressArg] = Builder.CreateAdd(Args[AddressArg], Tmp);
964 for (
unsigned I = 0;
I < StoreNum; ++
I) {
965 Mask.push_back(
Base +
I);
970 SubVal = Builder.CreateShuffleVector(Val, Mask);
972 Args.push_back(SubVal);
974 Builder.CreateIntrinsic(Builder.getVoidTy(), StoreIntrinsic, Args);
976 ExtractNum -= StoreNum;
984 if (ClampIntrinsic == Intrinsic::dx_uclamp)
985 return Intrinsic::umax;
986 if (ClampIntrinsic == Intrinsic::dx_sclamp)
987 return Intrinsic::smax;
988 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
989 return Intrinsic::maxnum;
993 if (ClampIntrinsic == Intrinsic::dx_uclamp)
994 return Intrinsic::umin;
995 if (ClampIntrinsic == Intrinsic::dx_sclamp)
996 return Intrinsic::smin;
997 assert(ClampIntrinsic == Intrinsic::dx_nclamp);
998 return Intrinsic::minnum;
1006 Type *Ty =
X->getType();
1008 auto *MaxCall = Builder.CreateIntrinsic(Ty,
getMaxForClamp(ClampIntrinsic),
1009 {
X, Min},
nullptr,
"dx.max");
1010 return Builder.CreateIntrinsic(Ty,
getMinForClamp(ClampIntrinsic),
1011 {MaxCall, Max},
nullptr,
"dx.min");
1016 Type *Ty =
X->getType();
1026 GT = Builder.CreateFCmpOLT(
Zero,
X);
1027 LT = Builder.CreateFCmpOLT(
X,
Zero);
1030 GT = Builder.CreateICmpSLT(
Zero,
X);
1031 LT = Builder.CreateICmpSLT(
X,
Zero);
1034 Value *ZextGT = Builder.CreateZExt(GT, RetTy);
1035 Value *ZextLT = Builder.CreateZExt(LT, RetTy);
1037 return Builder.CreateSub(ZextGT, ZextLT);
1049 bool IsDouble = Ty->getScalarType()->isDoubleTy();
1050 unsigned BitWidth = IsDouble ? 32 : Ty->getScalarSizeInBits();
1053 auto CopySignBit = [&](
Value *MagnitudeInt,
Value *SignInt) {
1056 Constant *SignMask = ConstantInt::get(IntTy, SignMaskVal);
1057 Constant *NotSignMask = ConstantInt::get(IntTy, ~SignMaskVal);
1059 Value *MagnitudeBits = Builder.CreateAnd(MagnitudeInt, NotSignMask);
1060 Value *SignBits = Builder.CreateAnd(SignInt, SignMask);
1061 return Builder.CreateOr(MagnitudeBits, SignBits);
1067 Value *MagnitudeHalves = Builder.CreateIntrinsic(
1068 SplitTy, Intrinsic::dx_splitdouble, {Magnitude});
1070 Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble, {
Sign});
1071 Value *MagnitudeLow = Builder.CreateExtractValue(MagnitudeHalves, 0);
1072 Value *MagnitudeHigh = Builder.CreateExtractValue(MagnitudeHalves, 1);
1073 Value *SignHigh = Builder.CreateExtractValue(SignHalves, 1);
1075 Value *CombinedHigh = CopySignBit(MagnitudeHigh, SignHigh);
1076 return Builder.CreateIntrinsic(Ty, Intrinsic::dx_asdouble,
1077 {MagnitudeLow, CombinedHigh});
1080 Value *MagnitudeInt = Builder.CreateBitCast(Magnitude, IntTy);
1081 Value *SignInt = Builder.CreateBitCast(
Sign, IntTy);
1082 Value *CombinedInt = CopySignBit(MagnitudeInt, SignInt);
1083 return Builder.CreateBitCast(CombinedInt, Ty);
1098 Type *EltTy = RetTy->getElementType();
1109 unsigned LHSSize = LHSRows * LHSCols;
1110 unsigned RHSSize = LHSCols * RHSCols;
1113 for (
unsigned I = 0;
I < LHSSize; ++
I)
1114 LHSElts[
I] = Builder.CreateExtractElement(
LHS,
I);
1115 for (
unsigned I = 0;
I < RHSSize; ++
I)
1116 RHSElts[
I] = Builder.CreateExtractElement(
RHS,
I);
1121 bool UseScalarFP = IsFP && (EltTy->
isDoubleTy() || LHSCols == 1);
1122 if (IsFP && !UseScalarFP) {
1125 FloatDotID = Intrinsic::dx_dot2;
1128 FloatDotID = Intrinsic::dx_dot3;
1131 FloatDotID = Intrinsic::dx_dot4;
1135 "Invalid matrix inner dimension for dot product: must be 2-4");
1140 for (
unsigned C = 0;
C < RHSCols; ++
C) {
1141 for (
unsigned R = 0; R < LHSRows; ++R) {
1144 for (
unsigned K = 0;
K < LHSCols; ++
K) {
1152 Dot = Builder.CreateFMul(RowElts[0], ColElts[0]);
1153 for (
unsigned K = 1;
K < LHSCols; ++
K)
1154 Dot = Builder.CreateIntrinsic(EltTy, Intrinsic::fmuladd,
1155 {RowElts[K], ColElts[K], Dot});
1159 Args.append(RowElts.
begin(), RowElts.
end());
1160 Args.append(ColElts.
begin(), ColElts.
end());
1161 Dot = Builder.CreateIntrinsic(EltTy, FloatDotID, Args);
1164 Dot = Builder.CreateMul(RowElts[0], ColElts[0]);
1165 for (
unsigned K = 1;
K < LHSCols; ++
K)
1166 Dot = Builder.CreateIntrinsic(EltTy, Intrinsic::dx_imad,
1167 {RowElts[K], ColElts[K], Dot});
1169 unsigned ResIdx =
C * LHSRows + R;
1170 Result = Builder.CreateInsertElement(Result, Dot, ResIdx);
1184 unsigned NumElts = Rows * Cols;
1186 for (
unsigned I = 0;
I < NumElts; ++
I)
1187 Mask[
I] = (
I % Cols) * Rows + (
I / Cols);
1190 return Builder.CreateShuffleVector(Mat, Mask);
1205 Type *ScalarTy = VT->getElementType();
1206 unsigned NumElems = VT->getNumElements();
1212 Value *StartColI32 = Builder.CreateZExt(StartCol, Int32Ty);
1215 M, Intrinsic::dx_store_output, {ScalarTy});
1217 for (
unsigned I = 0;
I < NumElems; ++
I) {
1219 Builder.CreateExtractElement(
Data, ConstantInt::get(Int32Ty,
I));
1221 Builder.CreateAdd(StartColI32, ConstantInt::get(Int32Ty,
I));
1222 Value *ColI8 = Builder.CreateTrunc(ColIdx, Int8Ty);
1223 Builder.CreateCall(ScalarFn, {SigElementId, RowIndex, ColI8,
Scalar});
1241 Type *ScalarTy = VT->getElementType();
1242 unsigned NumElems = VT->getNumElements();
1248 Value *StartColI32 = Builder.CreateZExt(StartCol, Int32Ty);
1251 M, Intrinsic::dx_load_input, {ScalarTy});
1254 for (
unsigned I = 0;
I < NumElems; ++
I) {
1256 Builder.CreateAdd(StartColI32, ConstantInt::get(Int32Ty,
I));
1257 Value *ColI8 = Builder.CreateTrunc(ColIdx, Int8Ty);
1259 ScalarFn, {SigElementId, RowIndex, ColI8, GsVertexOrPrimIndex});
1261 Builder.CreateInsertElement(Vec,
Scalar, ConstantInt::get(Int32Ty,
I));
1273 Builder.CreateSExt(Orig->
getArgOperand(0), Builder.getInt32Ty());
1276 {Intrinsic::dx_quad_read_across_x, Intrinsic::dx_quad_read_across_y,
1277 Intrinsic::dx_quad_read_across_diagonal}) {
1278 Value *QuadValue = Builder.CreateIntrinsic(ID, {Builder.getInt32Ty()},
Cond,
1279 nullptr,
"", Bundles);
1280 Result = Builder.CreateAnd(Result, QuadValue);
1282 return Builder.CreateTrunc(Result, Builder.getInt1Ty());
1286 Value *Result =
nullptr;
1288 switch (IntrinsicId) {
1289 case Intrinsic::abs:
1292 case Intrinsic::assume:
1295 case Intrinsic::atan2:
1298 case Intrinsic::copysign:
1301 case Intrinsic::fshl:
1304 case Intrinsic::fshr:
1307 case Intrinsic::exp:
1310 case Intrinsic::is_fpclass:
1313 case Intrinsic::log:
1316 case Intrinsic::log10:
1319 case Intrinsic::pow:
1320 case Intrinsic::powi:
1323 case Intrinsic::vector_reduce_and:
1324 case Intrinsic::vector_reduce_or:
1327 case Intrinsic::dx_uclamp:
1328 case Intrinsic::dx_sclamp:
1329 case Intrinsic::dx_nclamp:
1332 case Intrinsic::dx_isfinite:
1335 case Intrinsic::dx_isinf:
1338 case Intrinsic::dx_isnan:
1341 case Intrinsic::dx_fdot:
1344 case Intrinsic::dx_sdot:
1345 case Intrinsic::dx_udot:
1348 case Intrinsic::dx_sign:
1351 case Intrinsic::dx_quad_all:
1354 case Intrinsic::dx_load_input:
1357 case Intrinsic::dx_store_output:
1361 case Intrinsic::dx_resource_load_rawbuffer:
1365 case Intrinsic::dx_resource_store_rawbuffer:
1369 case Intrinsic::dx_resource_load_typedbuffer:
1373 case Intrinsic::dx_resource_store_typedbuffer:
1377 case Intrinsic::usub_sat:
1380 case Intrinsic::umul_with_overflow:
1381 case Intrinsic::smul_with_overflow:
1383 Intrinsic::smul_with_overflow);
1385 case Intrinsic::vector_reduce_add:
1386 case Intrinsic::vector_reduce_fadd:
1389 case Intrinsic::matrix_multiply:
1392 case Intrinsic::matrix_transpose:
1408 bool IntrinsicExpanded =
false;
1415 if (
F.user_empty() && IntrinsicExpanded)
1416 F.eraseFromParent();
1435 "DXIL Intrinsic Expansion",
false,
false)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file implements a class to represent arbitrary precision integral constant values and operations...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static Value * expand16BitIsNormal(CallInst *Orig)
static Value * createMulHighUnsigned(IRBuilder<> &Builder, Value *A, Value *B, Type *Ty, unsigned BW)
static bool expandIntrinsic(Function &F, CallInst *Orig)
static Value * expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic)
static Value * expand16BitIsInf(CallInst *Orig)
static bool expansionIntrinsics(Module &M)
static Value * expandCopySignIntrinsic(CallInst *Orig)
static Value * expand16BitIsFinite(CallInst *Orig)
static Value * expandLoadInput(CallInst *Orig)
static Value * expandUsubSat(CallInst *Orig)
static Value * expandFloatDotIntrinsic(CallInst *Orig)
static bool expandStoreOutput(CallInst *Orig)
static Value * expandMatrixTranspose(CallInst *Orig)
static Value * expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId)
static Value * expandAtan2Intrinsic(CallInst *Orig)
static Value * expandQuadAll(CallInst *Orig)
static Value * expandLog10Intrinsic(CallInst *Orig)
static Intrinsic::ID getMinForClamp(Intrinsic::ID ClampIntrinsic)
static Value * expandIntegerDotIntrinsic(CallInst *Orig, Intrinsic::ID DotIntrinsic)
static Value * expandVecReduceAndOr(CallInst *Orig, Intrinsic::ID IntrinsicId)
static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw)
static Value * expandLogIntrinsic(CallInst *Orig, float LogConstVal=numbers::ln2f)
static Value * expandMulWithOverflow(CallInst *Orig, bool Signed)
static Value * expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId)
static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy, bool IsRaw)
static Value * expandExpIntrinsic(CallInst *Orig)
static Value * expand16BitIsNaN(CallInst *Orig)
static Value * expandSignIntrinsic(CallInst *Orig)
static Intrinsic::ID getMaxForClamp(Intrinsic::ID ClampIntrinsic)
static bool shouldExpandFloatDotIntrinsic(Function &F)
static Value * expandFloatDotChunk(CallInst *Orig, Value *A, Value *B)
static Value * expandAbs(CallInst *Orig)
static bool isIntrinsicExpansion(Function &F)
static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw)
static Value * expandMatrixMultiply(CallInst *Orig)
static Value * expandIsFPClass(CallInst *Orig)
static Value * expandFunnelShiftIntrinsic(CallInst *Orig)
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
SmallVector< MachineOperand, 4 > Cond
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
DXILIntrinsicExpansionLegacy()
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
void setAttributes(AttributeList A)
Set the attributes for this call.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
AttributeList getAttributes() const
Return the attributes for this call.
This class represents a function call, abstracting a target machine's calling convention.
void setTailCall(bool IsTc=true)
static LLVM_ABI Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Type * getParamType(unsigned i) const
Parameter type accessors.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
iterator_range< user_iterator > users()
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
A Module instance is used to store all the information related to an LLVM module.
const Triple & getTargetTriple() const
Get the target triple which is a string describing the target host.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
LLVM_ABI VersionTuple getOSVersion() const
Parse the version number from the OS name component of the triple, if present.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI Type * getStructElementType(unsigned N) const
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVM_ABI Type * getWithNewBitWidth(unsigned NewBitWidth) const
Given an integer or vector type, change the lane bitwidth to NewBitwidth, whilst keeping the old numb...
static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
LLVM_ABI Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Represents a version number in the form major[.minor[.subminor[.build]]].
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
@ Load
The value being inserted comes from a load (InsertElement only).
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
ModulePass * createDXILIntrinsicExpansionLegacyPass()
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
@ Sub
Subtraction of integers.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.